diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 33092e163bfe1bf6c469244e2a634f15f5d82ff7..555603e6b2eb590f0085d6b0db59ec78ff85686a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,13 +23,14 @@ variables: PRM_FILES: "scripts/config/self_test.prm scripts/config/self_test_ltv.prm" TESTCASE_TIMEOUT_STV: 900 TESTCASE_TIMEOUT_LTV: 2400 - TESTCASE_TIMEOUT_LTV_SANITIZERS: 6000 + TESTCASE_TIMEOUT_STV_SANITIZERS: 1800 CI_REGRESSION_THRESH_MLD: "0.1" CI_REGRESSION_THRESH_MAX_ABS_DIFF: "50" CI_REGRESSION_THRESH_SSNR: "-1" CI_REGRESSION_THRESH_ODG: "-0.05" GIT_CLEAN_FLAGS: -ffdxq INSTR_DIR: "scripts/c-code_instrument" + TESTS_DIR_CODEC_BE_ON_MR: "tests/codec_be_on_mr_nonselection" BUILD_WITH_DEBUG_MODE_INFO: "" ENCODER_TEST: "" MANUAL_PIPELINE_TYPE: @@ -105,6 +106,8 @@ stages: - prevalidate - build - test + - compare + - postvalidate - deploy # --------------------------------------------------------------- @@ -121,6 +124,70 @@ stages: echo "Commit time was $CI_COMMIT_TIMESTAMP" date | xargs echo "System time is" +.print-common-info-windows: &print-common-info-windows + - | + echo "Printing common information for build job." + echo "Current job is run on commit $CI_COMMIT_SHA" + echo "Commit time was $CI_COMMIT_TIMESTAMP" + ("echo 'System time is'", "Get-Date -Format 'dddd dd/MM/yyyy HH:mm K'") | Invoke-Expression + +.disable-debugging-macro: &disable-debugging-macro + # automatically disable #DEBUGGING macro in options.h using /**/-comment + - sed -i.bak -e "s/^[[:space:]]*\(#define[[:space:]]*DEBUGGING\)/\/\*\1\*\//g" lib_com/options.h + +# From float CI +.mr-fetch-target-branch: &mr-fetch-target-branch + # first delete local target branch to avoid conflicts when branch is cached and there are merge conflicts during fetching + # depending on chaching, the branch may not be there, so prevent failure of this command -> should maybe be done smarter later + - git branch -D $CI_MERGE_REQUEST_TARGET_BRANCH_NAME || true + # needed when depth is lower than the number of commits in the branch + - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:$CI_MERGE_REQUEST_TARGET_BRANCH_NAME + +# From float CI +.mr-get-target-commit: &mr-get-target-commit # compare to last target branch commit before pipeline was created + - target_commit=$(git log $CI_MERGE_REQUEST_TARGET_BRANCH_NAME -1 --oneline --before=${CI_PIPELINE_CREATED_AT} --format=%H) + +# From float CI +.merge-request-comparison-setup-codec: &merge-request-comparison-setup-codec + ### build test binaries, initial clean for paranoia reasons + - *disable-debugging-macro + - make clean + - make -j + - mv IVAS_cod IVAS_cod_test + - mv IVAS_dec IVAS_dec_test + - mv IVAS_rend IVAS_rend_test + - git restore . + + ### store the current commit hash + - source_branch_commit_sha=$(git rev-parse HEAD) + + ### checkout version to compare against + - *mr-fetch-target-branch + - *mr-get-target-commit + - git checkout $target_commit + - echo "Building reference codec at commit $target_commit" + + ### build reference binaries + - *disable-debugging-macro + - make -j + - mv IVAS_cod IVAS_cod_ref + - mv IVAS_dec IVAS_dec_ref + - mv IVAS_rend IVAS_rend_ref + - git restore . + + # rename test binaries back + - mv IVAS_cod_test IVAS_cod + - mv IVAS_dec_test IVAS_dec + - mv IVAS_rend_test IVAS_rend + +.merge-request-comparison-check: &merge-request-comparison-check + - echo "--------------- Running merge-request-comparison-check anchor ---------------" + - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -ne 0 ] && [ $non_be_flag == 0 ]; then echo "Non-bitexact cases without non-BE tag encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -ne 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 + + .activate-debug-mode-info-if-set: &activate-debug-mode-info-if-set - if [ "$BUILD_WITH_DEBUG_MODE_INFO" = "true" ]; then - sed -i.bak -e "s/\/\*\ *\(#define\ *DEBUGGING\ *\)\*\//\1/g" lib_com/options.h @@ -185,7 +252,20 @@ stages: - cd $LTV_DIR - git pull - cd - - + +.get-commits-behind-count: &get-commits-behind-count + - echo $CI_COMMIT_SHA + - echo $CI_MERGE_REQUEST_TARGET_BRANCH_NAME + - commits_behind_count=$(git rev-list --count $CI_COMMIT_SHA..origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME) + +.check-commits-behind-count-in-compare-jobs: &check-commits-behind-count-in-compare-jobs + - | + if [ $commits_behind_count -ne 0 ]; then + echo "Your branch is not up-to-date with main -> Compare tests will not run as they can contain false negatives this way." + echo "Main might have changed during your pipeline run. Run 'git pull origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME' to update." + exit 1 + fi + .copy-ltv-files-to-testv-dir: ©-ltv-files-to-testv-dir - cp "$LTV_DIR"/*.wav scripts/testv/ - cp "$LTV_DIR"/*.met scripts/testv/ @@ -194,6 +274,10 @@ stages: .activate-Werror-linux: &activate-Werror-linux - sed -i.bak "s/^# \(CFLAGS += -Werror\)/\1/" Makefile +.activate-WX-windows: &activate-WX-windows + - (Get-Content -Path "CMakeLists.txt") -replace '# \(add_compile_options\("\/WX"\)\)', '$1' | Set-Content -Path "CMakeLists.txt" + - Get-ChildItem -Path "Workspace_msvc" -Filter "*.vcxproj" | ForEach-Object { (Get-Content -Path $_.FullName) -replace 'false', 'true' | Set-Content -Path $_.FullName } + .rules-pytest-to-ref-short: rules: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare" @@ -211,12 +295,30 @@ stages: - if: $CI_PIPELINE_SOURCE == 'schedule' when: never +# TODO: only temporary as long the MR encoder tests should not compare to main +.rules-pytest-to-ref-enc-short-temp: + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare" + - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - if: $CI_PIPELINE_SOURCE == 'schedule' + when: never + .rules-pytest-to-main-short: rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main - if: $CI_PIPELINE_SOURCE == 'push' when: never +.rules-mr-to-main-or-main-pc-or-manual: + rules: + - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare" + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main-pc") + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + .rules-pytest-long: rules: - if: $PYTEST_MLD_LONG # Set by scheduled pipeline @@ -248,6 +350,46 @@ stages: # Job templates # --------------------------------------------------------------- +# When designing templates, try not to use too much inheritance and +# if multiple templates and extended on, remember that on conflict, +# latest overwrites the parameter. + +# templates for rules +.rules-basis: + rules: + - if: $MIRROR_ACCESS_TOKEN # Don't run in the mirror update pipeline (only then MIRROR_ACCESS_TOKEN is defined) + when: never + - if: $CI_PIPELINE_SOURCE == 'schedule' # Don't run in any scheduled pipelines by default (use schedule templates below to enable again for certain conditions) + when: never + - if: $CI_PIPELINE_SOURCE == 'trigger' # Don't run triggered pipeline by default + when: never + - if: $MANUAL_PIPELINE_TYPE == 'test-be-release' # Skip all the normal jobs when testing manually against release codec + when: never + - if: $MANUAL_PIPELINE_TYPE == 'test-long-self-test' # Skip all the normal jobs when testing manually against release codec + when: never + - if: $MANUAL_PIPELINE_TYPE == 'ivas-conformance' + when: never + - if: $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' + when: never + - if: $MANUAL_PIPELINE_TYPE == 'check-clipping' + - if: $MANUAL_PIPELINE_TYPE == 'test-branch-vs-input-passthrough' + when: never + - when: on_success + +.rules-merge-request: + extends: .rules-basis + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + +.rules-merge-request-to-main-pc: + extends: .rules-basis + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'main-pc' + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + # templates to define stages and platforms .test-job-linux: tags: @@ -260,6 +402,13 @@ stages: tags: - ivas-basop-linux +.build-job-windows: + stage: build + needs: [] + timeout: "4 minutes" + tags: + - ivas-windows + # template for test jobs on linux that need the TESTV_DIR .test-job-linux-needs-testv-dir: extends: .test-job-linux @@ -372,7 +521,6 @@ stages: IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME" SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html" script: - - set -euxo pipefail - *print-common-info - *update-scripts-repo - if [ $USE_LTV -eq 1 ]; then @@ -410,7 +558,7 @@ stages: ### run main now - git checkout $CI_MERGE_REQUEST_TARGET_BRANCH_NAME - - git pull + - git pull origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME - make clean - make -j - python3 -m pytest --tb=no $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true @@ -420,10 +568,7 @@ stages: - diff_sba=0 - diff_param=0 - diff_report=0 - # SHORT_TEST_SUITE_ENCODER does not contain test_sba.py. This leads to non-existing output folders being compared and to diff_sba=1. Therefore, this is skipped for the encoder tests - - if [ "$TEST_SUITE" != "$SHORT_TEST_SUITE_ENCODER" ]; then - - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/sba_bs/raw tests/dut_branch/sba_bs/raw || diff_sba=$? - - fi + - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/sba_bs/raw tests/dut_branch/sba_bs/raw || diff_sba=$? - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/param_file/dec tests/dut_branch/param_file/dec || diff_param=$? - diff $CSV_BRANCH $CSV_MAIN || diff_report=$? - if [ $diff_param -eq 0 ] && [ $diff_sba -eq 0 ] && [ $diff_report -eq 0 ] && [ $zero_errors_branch -eq 1 ]; then @@ -439,11 +584,11 @@ stages: ### compare the two csv files for regressions - regressions_found=0 - - python3 scripts/basop_check_for_changes_in_testcases.py --xml_report $XML_REPORT_BRANCH $CSV_MAIN $CSV_BRANCH || regressions_found=$? + - python3 scripts/basop_check_for_changes_in_testcases.py $CSV_BRANCH $CSV_MAIN || regressions_found=$? - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit_code=$EXIT_CODE_NON_BE; fi - if [ $zero_errors_branch != 1 ]; then echo "Run errors encountered!"; exit_code=$EXIT_CODE_NON_BE; fi - - if [ $regressions_found != 0 ] && [ "$SKIP_REGRESSION_CHECK" != "true" ]; then + - if [ $regressions_found != 0 ]; then - if [ $allow_regressions_flag == 0 ]; then - echo "Detected regression wrt to main, [allow regression] not set!" - exit_code=$EXIT_CODE_FAIL; @@ -498,13 +643,12 @@ stages: script: - *print-common-info - *update-scripts-repo - - *copy-ltv-files-to-testv-dir - python3 ci/remove_unsupported_testcases.py $PRM_FILES - *build-reference-and-dut-binaries - make clean - make -j CLANG=$CLANG_NUM - if [[ $CLANG_NUM == 3 ]]; then export UBSAN_OPTIONS="suppressions=scripts/ubsan.supp,report_error_type=1"; fi - - testcase_timeout=$TESTCASE_TIMEOUT_LTV_SANITIZERS + - testcase_timeout=$TESTCASE_TIMEOUT_STV_SANITIZERS - python3 -m pytest $TEST_SUITE -v --tb=no --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout $testcase_timeout --ref_encoder_path $DUT_ENCODER_PATH --ref_decoder_path $DUT_DECODER_PATH artifacts: name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" @@ -603,6 +747,42 @@ uninterruptible: tags: - ivas-basop-linux +# --------------------------------------------------------------- +# Validation jobs +# --------------------------------------------------------------- + +branch-is-up-to-date-with-target-pre: + extends: + - .rules-merge-request + stage: prevalidate + needs: [] + tags: + - ivas-basop-linux + script: + - *get-commits-behind-count + - echo $commits_behind_count + - | + if [ $commits_behind_count -ne 0 ]; then + echo "Your branch is behind the target branch, run 'git pull origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME' to update." + exit 1 + fi + +branch-is-up-to-date-with-target-post: + extends: + - .rules-merge-request + stage: postvalidate + tags: + - ivas-basop-linux + script: + - *get-commits-behind-count + - echo $commits_behind_count + - | + if [ $commits_behind_count -ne 0 ]; then + echo "Your branch is behind the target branch, possibly main changed during your pipeline run, run 'git pull origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME' to update." + exit 1 + fi + + # --------------------------------------------------------------- # verification jobs # --------------------------------------------------------------- @@ -611,7 +791,7 @@ clang-format-check: extends: - .test-job-linux rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - if: $CI_PIPELINE_SOURCE == 'push' when: never - if: $CI_PIPELINE_SOURCE == 'schedule' @@ -619,6 +799,8 @@ clang-format-check: variables: ARTIFACT_BASE_NAME: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--formatting-fix" stage: prevalidate + tags: + - ivas-basop-linux needs: [] timeout: "5 minutes" script: @@ -659,6 +841,32 @@ clang-format-check: name: "$ARTIFACT_BASE_NAME" expose_as: "formatting patch" +# from float +# needs fix to be merged from floating point repo before it can be activated +.check-bitexactness-hrtf-rom-and-file: + extends: + - .test-job-linux + - .rules-merge-request-to-main-pc + stage: test + needs: ["build-codec-linux-make"] + timeout: "5 minutes" + script: + - *print-common-info + - *update-scripts-repo + - make clean + - make -j + - python3 tests/create_short_testvectors.py --cut_len 1.0 + # TODO: run full test again once the custom binary files are supported + - python3 -m pytest -k "not diff_from_rom and not test_binary_file" tests/hrtf_binary_loading --html=report.html --junit-xml=report-junit.xml --self-contained-html + artifacts: + paths: + - report.html + - report-junit.xml + when: always + name: "$CI_JOB_NAME--$CI_MERGE_REQUEST_ID--sha-$CI_COMMIT_SHA--hrtf-loading" + expose_as: "logs-hrtf-loading" + expire_in: "5 days" + # --------------------------------------------------------------- # Build jobs # --------------------------------------------------------------- @@ -674,6 +882,8 @@ build-codec-linux-make: when: never extends: - .build-job-linux + tags: + - ivas-basop-linux script: - *print-common-info - *activate-Werror-linux @@ -684,19 +894,55 @@ build-codec-linux-instrumented-make: rules: - if: $CI_PIPELINE_SOURCE == 'web' - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - if: $CI_PIPELINE_SOURCE == 'schedule' - if: $CI_PIPELINE_SOURCE == 'push' when: never extends: - .build-job-linux timeout: "7 minutes" + tags: + - ivas-basop-linux script: - *print-common-info - *update-scripts-repo - bash scripts/prepare_instrumentation.sh -m MEM_ONLY -p BASOP - make -j -C $INSTR_DIR + +# make sure that the codec builds with msan, asan and usan +build-codec-sanitizers-linux: + extends: + - .build-job-linux + - .rules-basis + tags: + - ivas-basop-linux + script: + - *update-scripts-repo + - *print-common-info + # TODO: re-enable once all the warnings have been fixed + #- *activate-Werror-linux + - bash ci/build_codec_sanitizers_linux.sh + +build-codec-windows-msbuild: + rules: + - if: $CI_PIPELINE_SOURCE == 'web' + - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' # trigger build job for all MRs + - if: $CI_PIPELINE_SOURCE == 'schedule' + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + extends: + - .build-job-windows + timeout: "7 minutes" + tags: + - ivas-windows + script: + - *print-common-info-windows + - *activate-WX-windows + - MSBuild.exe -maxcpucount .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug + + build-codec-linux-debugging-make: rules: - if: $CI_PIPELINE_SOURCE == 'web' @@ -720,46 +966,44 @@ build-codec-linux-debugging-make: # --------------------------------------------------------------- ### jobs that test fx encoder -> flt decoder -ivas-pytest-compare_to_main-short-enc: - extends: - - .rules-pytest-to-main-short - - .test-job-linux - before_script: - - USE_LTV=0 - - DUT_DECODER_PATH=./IVAS_dec_ref - - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - - LEVEL_SCALING=1.0 - - SKIP_REGRESSION_CHECK="true" - <<: *ivas-pytest-on-merge-request-anchor - -ivas-pytest-compare_to_main-short-enc-lev-10: - extends: - - .rules-pytest-to-main-short - - .test-job-linux - before_script: - - USE_LTV=0 - - DUT_DECODER_PATH=./IVAS_dec_ref - - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - - LEVEL_SCALING=0.3162 - - SKIP_REGRESSION_CHECK="true" - <<: *ivas-pytest-on-merge-request-anchor - -ivas-pytest-compare_to_main-short-enc-lev+10: - extends: - - .rules-pytest-to-main-short - - .test-job-linux - before_script: - - USE_LTV=0 - - DUT_DECODER_PATH=./IVAS_dec_ref - - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - - LEVEL_SCALING=3.162 - - SKIP_REGRESSION_CHECK="true" - <<: *ivas-pytest-on-merge-request-anchor +# TODO: reenable once encoder tests shall compare to main +# ivas-pytest-compare_to_main-short-enc: +# extends: +# - .rules-pytest-to-main-short +# - .test-job-linux +# before_script: +# - USE_LTV=0 +# - DUT_DECODER_PATH=./IVAS_dec_ref +# - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" +# - LEVEL_SCALING=1.0 +# <<: *ivas-pytest-on-merge-request-anchor + +# ivas-pytest-compare_to_main-short-enc-lev-10: +# extends: +# - .rules-pytest-to-main-short +# - .test-job-linux +# before_script: +# - USE_LTV=0 +# - DUT_DECODER_PATH=./IVAS_dec_ref +# - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" +# - LEVEL_SCALING=0.3162 +# <<: *ivas-pytest-on-merge-request-anchor + +# ivas-pytest-compare_to_main-short-enc-lev+10: +# extends: +# - .rules-pytest-to-main-short +# - .test-job-linux +# before_script: +# - USE_LTV=0 +# - DUT_DECODER_PATH=./IVAS_dec_ref +# - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" +# - LEVEL_SCALING=3.162 +# <<: *ivas-pytest-on-merge-request-anchor ### jobs that test flt encoder -> fx decoder ivas-pytest-compare_to_main-short-dec: extends: - - .rules-pytest-to-main-short + - .rules-mr-to-main-or-main-pc-or-manual - .test-job-linux before_script: - USE_LTV=0 @@ -771,7 +1015,7 @@ ivas-pytest-compare_to_main-short-dec: ivas-pytest-compare_to_main-short-dec-lev-10: extends: - - .rules-pytest-to-main-short + - .rules-mr-to-main-or-main-pc-or-manual - .test-job-linux before_script: - USE_LTV=0 @@ -783,7 +1027,7 @@ ivas-pytest-compare_to_main-short-dec-lev-10: ivas-pytest-compare_to_main-short-dec-lev+10: extends: - - .rules-pytest-to-main-short + - .rules-mr-to-main-or-main-pc-or-manual - .test-job-linux before_script: - USE_LTV=0 @@ -800,7 +1044,8 @@ ivas-pytest-compare_to_main-short-dec-lev+10: ### jobs that test fx encoder -> flt decoder ivas-pytest-compare_to_ref-short-enc: extends: - - .rules-pytest-to-ref-short + #- .rules-pytest-to-ref-short + - .rules-mr-to-main-or-main-pc-or-manual - .test-job-linux before_script: - USE_LTV=0 @@ -812,8 +1057,9 @@ ivas-pytest-compare_to_ref-short-enc: ivas-pytest-compare_to_ref-short-enc-lev-10: extends: - - .rules-pytest-to-ref-short - - .test-job-linux + #- .rules-pytest-to-ref-short + - .rules-mr-to-main-or-main-pc-or-manual + - .test-job-linux before_script: - USE_LTV=0 - ENCODER_TEST="true" @@ -824,8 +1070,9 @@ ivas-pytest-compare_to_ref-short-enc-lev-10: ivas-pytest-compare_to_ref-short-enc-lev+10: extends: - - .rules-pytest-to-ref-short - - .test-job-linux + #- .rules-pytest-to-ref-short + - .rules-mr-to-main-or-main-pc-or-manual + - .test-job-linux before_script: - USE_LTV=0 - ENCODER_TEST="true" @@ -924,7 +1171,7 @@ ivas-pytest-dec-msan: before_script: - CLANG_NUM=1 - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER + - TEST_SUITE=$LONG_TEST_SUITE_SELF_TEST <<: *ivas-pytest-sanitizers-anchor ivas-pytest-dec-asan: @@ -933,7 +1180,7 @@ ivas-pytest-dec-asan: before_script: - CLANG_NUM=2 - DUT_ENCODER_PATH=./$REF_ENCODER_PATH - - TEST_SUITE=$LONG_TEST_SUITE_NO_RENDERER + - TEST_SUITE=$LONG_TEST_SUITE_SELF_TEST <<: *ivas-pytest-sanitizers-anchor ivas-pytest-dec-usan: @@ -1121,7 +1368,7 @@ be-2-evs-26444: - .test-job-linux rules: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "evs-26444" - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" || $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main-pc") tags: - be-2-evs-basop stage: test @@ -1186,6 +1433,146 @@ voip-be-on-merge-request: - python3 -m pytest tests/test_be_for_jbm_neutral_dly_profile.py +# --------------------------------------------------------------- +# Test jobs for merge requests +# --------------------------------------------------------------- + +# test that runs all modes with 1s input signals +# TODO: disabled temporarily, needs to be adapted to BASOP +.codec-smoke-test: + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request + timeout: "20 minutes" + tags: + - ivas-basop-linux + stage: test + needs: ["build-codec-linux-make"] #, "build-codec-instrumented-linux", "build-codec-sanitizers-linux"] + script: + - *print-common-info + # LTV update needed as ltv ISM metadata files are used + - *update-ltv-repo + - bash ci/smoke_test.sh + ### analyze for failures + - if ! [ -s smoke_test_output.txt ] || ! [ -s smoke_test_output_jbm.txt ] || ! [ -s smoke_test_output_hrtf.txt ]; then echo "Error in smoke test"; exit 1; fi + - ret_val=0 + - if cat smoke_test_output.txt | grep -c "failed"; then echo "Smoke test without JBM failed"; ret_val=1; fi + - if cat smoke_test_output_jbm.txt | grep -c "failed"; then echo "Smoke test JBM part failed"; ret_val=1; fi + - if cat smoke_test_output_hrtf.txt | grep -c "failed"; then echo "Smoke test with external hrtf files failed"; ret_val=1; fi + - exit $ret_val + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + expire_in: 1 week + when: always + paths: + - smoke_test_output.txt + - smoke_test_output_jbm.txt + - smoke_test_output_hrtf.txt + expose_as: "Smoke test results" + +# compare renderer bitexactness between target and source branch +renderer-pytest-on-merge-request: + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request-to-main-pc + needs: ["build-codec-linux-make"] + # TODO: set reasonable timeout, will most likely take less + timeout: "20 minutes" + tags: + - ivas-basop-linux + stage: compare + script: + - *print-common-info + - *get-commits-behind-count + - *check-commits-behind-count-in-compare-jobs + - *merge-request-comparison-setup-codec + + # some helper variables - "|| true" to prevent failures from grep not finding anything + # write to temporary file as workaround for failures observed with piping echo + - echo $CI_MERGE_REQUEST_TITLE > tmp.txt + - non_be_flag=$(grep -c --ignore-case "\[rend\(erer\)*[ -]*non[ -]*be\]" tmp.txt) || true + # TODO: ref_using_target comes from float repo, but does not apply here - disable for now + # - ref_using_target=$(grep -c --ignore-case "\[ref[ -]*using[ -]*target\]" tmp.txt) || true + - ref_using_target=0 + + ### If ref_using_target is not set, checkout the source branch to use scripts and input from there + - if [ $ref_using_target == 0 ]; then git checkout $source_branch_commit_sha; fi + + - exit_code=0 + - testcase_timeout=60 + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py --create_ref --testcase_timeout=$testcase_timeout || exit_code=$? + + ### Run test using branch scripts and input + - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi + + # run test + - python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + + - *merge-request-comparison-check + +# compare bit exactness between target and source branch +ivas-pytest-on-merge-request: + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request-to-main-pc + stage: compare + # TODO: broken dependency needs to be removed temporarily, see above + # note: this step doesn't really depend on codec-smoke-test + # it's just pointless to run this step when the smoke test fails and the smoke test should be reasonably fast + # thus, overall, this should save time + needs: ["build-codec-linux-make"] #, "codec-smoke-test"] + timeout: "40 minutes" + tags: + - ivas-basop-linux + script: + - *print-common-info + - *get-commits-behind-count + - *check-commits-behind-count-in-compare-jobs + - *merge-request-comparison-setup-codec + - python3 ci/remove_unsupported_testcases.py $PRM_FILES + + # some helper variables - "|| true" to prevent failures from grep not finding anything + # write to temporary file as workaround for failures observed with piping echo + - echo $CI_MERGE_REQUEST_TITLE > tmp.txt + - non_be_flag=$(grep -c --ignore-case "\[non[ -]*be\]" tmp.txt) || true + # TODO: ref_using_target comes from float repo, but does not apply here - disable for now + # - ref_using_target=$(grep -c --ignore-case "\[ref[ -]*using[ -]*target\]" tmp.txt) || true + - ref_using_target=0 + + ### If ref_using_target is not set, checkout the source branch to use scripts and input from there + - if [ $ref_using_target == 0 ]; then git checkout $source_branch_commit_sha; fi + + ### prepare pytest + # create references + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 + + ### Run test using branch scripts and input + - if [ $ref_using_target == 1 ]; then git checkout $source_branch_commit_sha; fi + + ### run pytest + - exit_code=0 + - testcase_timeout=600 + - python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout || exit_code=$? + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + + - *merge-request-comparison-check + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + expire_in: 1 week + when: always + paths: + - report-junit.xml + - report.html + expose_as: "pytest ivas results" + reports: + junit: + - report-junit.xml + # --------------------------------------------------------------- # Complexity measurement jobs # --------------------------------------------------------------- @@ -1197,9 +1584,6 @@ voip-be-on-merge-request: - job_id=$(python3 ci/get_id_of_last_job_occurence.py $CI_COMMIT_REF_NAME $CI_JOB_NAME $CI_PROJECT_ID) - echo $job_id - # this is a testing/maintenance mechanism to force getting the log history from a specific job id - # see below in the concrete complexity jobs - - if [ "$JOB_ID_INJECT" != "" ]; then job_id=$JOB_ID_INJECT; fi - curl --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts.zip - unzip artifacts.zip || true # this may fail on first run, when there are no artifacts there and the zip file is actually just "404"-html - ls @@ -1210,13 +1594,21 @@ voip-be-on-merge-request: - rm artifacts.zip - rm -rf $public_dir - ### 1.5.part: get the corresponding measurement from ivas-float-update - - job_id=$(python3 ci/get_id_of_last_job_occurence.py ivas-float-update $CI_JOB_NAME $CI_PROJECT_ID) + ### 1.5.part: get the corresponding measurement from ivas-float-update-cmplx + - job_id=$(python3 ci/get_id_of_last_job_occurence.py ivas-float-update-cmplx $CI_JOB_NAME $CI_PROJECT_ID) - echo $job_id - curl --request GET "https://forge.3gpp.org/rep/api/v4/projects/$CI_PROJECT_ID/jobs/$job_id/artifacts" --output artifacts_ref.zip - - unzip -j artifacts_ref.zip "*latest_WMOPS.csv" || true - # add file to arguments only if the artifact could be retrieved to prevent error later. - - if [ -f latest_WMOPS.csv ]; then GET_WMOPS_ARGS="$GET_WMOPS_ARGS latest_WMOPS.csv"; fi + - unzip -j artifacts_ref.zip "*latest_WMOPS.csv" + + ### 2. part: setup specific for BASOP repo + # hack for using the reference encoder -> need to build manually to make script use ref enc and BASOP dec + - mkdir COMPLEXITY + - cp IVAS_cod_ref COMPLEXITY/IVAS_cod + # build branch code aain with instrumentation + - make clean + - bash scripts/prepare_instrumentation.sh -p BASOP -m MEM_ONLY + - make -j -C $INSTR_DIR + - cp $INSTR_DIR/IVAS_dec COMPLEXITY/IVAS_dec .complexity-measurements-prepare-artifacts: &complexity-measurements-prepare-artifacts # prepare artifacts -> move to public directory @@ -1244,7 +1636,6 @@ voip-be-on-merge-request: stage: test variables: ret_val: 0 - GET_WMOPS_ARGS: "mem_only basop" timeout: 3 hours 30 minutes before_script: - *print-common-info @@ -1265,12 +1656,10 @@ complexity-stereo-in-stereo-out: - .complexity-template rules: - if: $MEASURE_COMPLEXITY_LINUX - variables: - JOB_ID_INJECT: "" script: - in_format=stereo - out_format=stereo - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val @@ -1281,13 +1670,11 @@ complexity-ism-in-binaural-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 1 hour - variables: - JOB_ID_INJECT: "" script: - in_format=ISM - out_format=BINAURAL - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val @@ -1298,13 +1685,11 @@ complexity-ism-in-binaural_room_ir-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 2 hours - variables: - JOB_ID_INJECT: "" script: - in_format=ISM - out_format=BINAURAL_ROOM_IR - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val @@ -1315,13 +1700,11 @@ complexity-ism-in-ext-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 3 hours 30 minutes - variables: - JOB_ID_INJECT: "" script: - in_format=ISM - out_format=EXT - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val @@ -1332,13 +1715,11 @@ complexity-sba-hoa3-in-hoa3-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 4 hours 30 minutes - variables: - JOB_ID_INJECT: "" script: - in_format=HOA3 - out_format=HOA3 - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val @@ -1349,13 +1730,11 @@ complexity-sba-hoa3-in-binaural-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 5 hours 30 minutes - variables: - JOB_ID_INJECT: "" script: - in_format=HOA3 - out_format=BINAURAL - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val @@ -1366,13 +1745,11 @@ complexity-sba-hoa3-in-binaural_room_ir-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 6 hours 30 minutes - variables: - JOB_ID_INJECT: "" script: - in_format=HOA3 - out_format=BINAURAL_ROOM_IR - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val @@ -1383,13 +1760,11 @@ complexity-mc-in-7_1_4-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 7 hours 30 minutes - variables: - JOB_ID_INJECT: "" script: - in_format=MC - out_format=7_1_4 - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val @@ -1400,13 +1775,11 @@ complexity-mc-in-binaural-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 10 hours - variables: - JOB_ID_INJECT: "" script: - in_format=MC - out_format=BINAURAL - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val @@ -1417,13 +1790,11 @@ complexity-mc-in-binaural_room_ir-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 12 hours 30 minutes - variables: - JOB_ID_INJECT: "" script: - in_format=MC - out_format=BINAURAL_ROOM_IR - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val @@ -1434,13 +1805,11 @@ complexity-masa-in-ext-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 15 hours - variables: - JOB_ID_INJECT: "" script: - in_format=MASA - out_format=EXT - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val @@ -1451,13 +1820,11 @@ complexity-masa-in-binaural-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 16 hours - variables: - JOB_ID_INJECT: "" script: - in_format=MASA - out_format=BINAURAL - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val @@ -1468,13 +1835,11 @@ complexity-masa-in-hoa3-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 17 hours - variables: - JOB_ID_INJECT: "" script: - in_format=MASA - out_format=HOA3 - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val @@ -1485,13 +1850,11 @@ complexity-masa-in-hoa3-out: # - if: $MEASURE_COMPLEXITY_LINUX # when: delayed # start_in: 13 hours - # variables: - # JOB_ID_INJECT: "" # script: # - in_format=OMASA # - out_format=EXT # - ret_val=0 -# - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? +# - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? # - *complexity-measurements-prepare-artifacts # - exit $ret_val @@ -1502,13 +1865,11 @@ complexity-omasa-in-binaural-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 18 hours - variables: - JOB_ID_INJECT: "" script: - in_format=OMASA - out_format=BINAURAL - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val @@ -1519,13 +1880,11 @@ complexity-omasa-in-hoa3-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 20 hours - variables: - JOB_ID_INJECT: "" script: - in_format=OMASA - out_format=HOA3 - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val @@ -1536,13 +1895,11 @@ complexity-StereoDmxEVS-stereo-in-mono-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 22 hours - variables: - JOB_ID_INJECT: "" script: - in_format=StereoDmxEVS - out_format=mono - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val @@ -1557,7 +1914,7 @@ complexity-StereoDmxEVS-stereo-in-mono-out: # - in_format=OSBA # - out_format=EXT # - ret_val=0 -# - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? +# - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? # - *complexity-measurements-prepare-artifacts # - exit $ret_val @@ -1568,13 +1925,11 @@ complexity-osba-in-binaural-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 22 hours 30 minutes - variables: - JOB_ID_INJECT: "" script: - in_format=OSBA - out_format=BINAURAL - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val @@ -1585,13 +1940,11 @@ complexity-osba-in-binaural_room_ir-out: - if: $MEASURE_COMPLEXITY_LINUX when: delayed start_in: 25 hours - variables: - JOB_ID_INJECT: "" script: - in_format=OSBA - out_format=BINAURAL_ROOM_IR - ret_val=0 - - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" $GET_WMOPS_ARGS || ret_val=$? + - bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format" mem_only basop latest_WMOPS.csv || ret_val=$? - *complexity-measurements-prepare-artifacts - exit $ret_val