diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c2e905ba6d99b64eb7bf9c28efffa43b5abebaf4..c51f8c54d0f8b5aa71b92a3af8a10f44323937d5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ variables: TESTV_DIR: "/usr/local/testv" LTV_DIR: "/usr/local/ltv" EVS_BE_TEST_DIR_BASOP: "/usr/local/be_2_evs_basop" - REFERENCE_BRANCH: "ivas-float-update" + FLOAT_REF_BRANCH: "ivas-float-update" BUILD_OUTPUT: "build_output.txt" SCRIPTS_DIR: "/usr/local/scripts" EXIT_CODE_NON_BE: 123 @@ -13,10 +13,20 @@ variables: SHORT_TEST_SUITE_ENCODER: "tests/codec_be_on_mr_nonselection/test_param_file.py --param_file scripts/config/self_test_basop_encoder.prm" LONG_TEST_SUITE_ENCODER: "tests/codec_be_on_mr_nonselection/test_param_file.py --param_file scripts/config/self_test_ltv_basop_encoder.prm" TEST_SUITE: "" + # These path variables are used by the pytest calls. + # They can be overwritten in the job templates to e.g. only test encoder or decoder in the chain DUT_ENCODER_PATH: "./IVAS_cod" DUT_DECODER_PATH: "./IVAS_dec" REF_ENCODER_PATH: "./IVAS_cod_ref" REF_DECODER_PATH: "./IVAS_dec_ref" + MERGE_TARGET_ENCODER_PATH: "./IVAS_cod_merge_target" + MERGE_TARGET_DECODER_PATH: "./IVAS_dec_merge_target" + # These path variables are used for building the binaries + # They should never be overwritten! + REF_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_cod_ref" + REF_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_dec_ref" + MERGE_TARGET_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_cod_merge_target" + MERGE_TARGET_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_dec_merge_target" LEVEL_SCALING: "1.0" IVAS_PIPELINE_NAME: '' BASOP_CI_BRANCH_PC_REPO: "basop-ci-branch" @@ -35,6 +45,12 @@ variables: DELTA_ODG: "" COMPARE_DMX: "" SKIP_REGRESSION_CHECK: "" + FAILED_TESTCASES_LIST: "failed-testcases.txt" + ERRORS_TESTCASES_LIST: "errors-testcases.txt" + PYTEST_CACHE_ARTIFACT: "pytest_cache.zip" + FLOAT_REF_COMMIT_FILE: "float-ref-git-sha" + CUT_COMMIT_FILE: "CuT-git-sha" + MERGE_TARGET_COMMIT_FILE: "merge-target-git-sha" MANUAL_PIPELINE_TYPE: description: "Type for the manual pipeline run. Use 'pytest-compare' to run comparison test against reference float codec." value: 'default' @@ -115,7 +131,9 @@ stages: - .pre - prevalidate - build + - check-be - test + - postvalidate - deploy # --------------------------------------------------------------- @@ -145,33 +163,49 @@ stages: - sed -i.bak -e "s/\/\*\ *\(#define\ *DEBUG_MODE_INFO\ *\)\*\//\1/g" lib_com/options.h - fi -.build-reference-binaries: &build-reference-binaries +.build-float-ref-binaries: &build-float-ref-binaries + - git rev-parse HEAD > $CUT_COMMIT_FILE - current_commit_sha=$(git rev-parse HEAD) ### build reference binaries - - git checkout $REFERENCE_BRANCH + - git checkout $FLOAT_REF_BRANCH - git pull - *activate-debug-mode-info-if-set - - cat lib_com/options.h - make clean - make -j - - mv ./IVAS_cod ./$REF_ENCODER_PATH - - mv ./IVAS_dec ./$REF_DECODER_PATH + - mv ./IVAS_cod ./$REF_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY + - mv ./IVAS_dec ./$REF_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY - mv ./IVAS_rend ./IVAS_rend_ref ### Return to current branch - git restore . + - git rev-parse HEAD > $FLOAT_REF_COMMIT_FILE - git checkout $current_commit_sha +.build-merge-target-binaries: &build-merge-target-binaries + - current_commit_sha=$(git rev-parse HEAD) + ### build merge target binaries + - git checkout $CI_MERGE_REQUEST_TARGET_BRANCH_NAME + - git pull + - *activate-debug-mode-info-if-set + - make clean + - make -j + - mv ./IVAS_cod ./$MERGE_TARGET_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY + - mv ./IVAS_dec ./$MERGE_TARGET_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY + - mv ./IVAS_rend ./IVAS_rend_merge_target + ### Return to current branch + - git restore . + - git rev-parse HEAD > $MERGE_TARGET_COMMIT_FILE + - git checkout $current_commit_sha -.build-reference-and-dut-binaries: &build-reference-and-dut-binaries +.build-float-ref-and-dut-binaries: &build-float-ref-and-dut-binaries ### build reference binaries - - *build-reference-binaries + - *build-float-ref-binaries ### build dut binaries - *activate-debug-mode-info-if-set - make clean - make -j -.build-and-create-reference-outputs: &build-and-create-reference-outputs - - *build-reference-and-dut-binaries +.build-and-create-float-ref-outputs: &build-and-create-float-ref-outputs + - *build-float-ref-and-dut-binaries ### prepare pytest # create short test vectors @@ -213,6 +247,11 @@ stages: - cd $LTV_DIR - git pull - cd - + +.get-commits-behind-count: &get-commits-behind-count + - echo $CI_COMMIT_SHA + - echo $CI_MERGE_REQUEST_TARGET_BRANCH_NAME + - commits_behind_count=$(git rev-list --count $CI_COMMIT_SHA..origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME) .copy-ltv-files-to-testv-dir: ©-ltv-files-to-testv-dir - cp "$LTV_DIR"/*.wav scripts/testv/ @@ -226,6 +265,13 @@ stages: - (Get-Content -Path "CMakeLists.txt") -replace '# \(add_compile_options\("\/WX"\)\)', '$1' | Set-Content -Path "CMakeLists.txt" - Get-ChildItem -Path "Workspace_msvc" -Filter "*.vcxproj" | ForEach-Object { (Get-Content -Path $_.FullName) -replace 'false', 'true' | Set-Content -Path $_.FullName } +.rules-merge-request: + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + - when: never + .rules-pytest-to-ref-short: rules: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare" @@ -296,17 +342,20 @@ stages: .build-job-linux: stage: build timeout: "2 minutes" - needs: [] tags: - ivas-basop-linux .build-job-windows: stage: build - needs: [] timeout: "4 minutes" tags: - ivas-windows +.print-results-banner: &print-results-banner + - set +x + - echo "" + - echo -e "==================================================================================================================\n================================================== TEST RESULTS ==================================================\n==================================================================================================================\n" + # template for test jobs on linux that need the TESTV_DIR .test-job-linux-needs-testv-dir: extends: .test-job-linux @@ -346,7 +395,7 @@ stages: - if [ "$COMPARE_DMX" = "true" ] || [ "$ENCODER_TEST" = "true" ]; then - BUILD_WITH_DEBUG_MODE_INFO="true" - fi - - *build-and-create-reference-outputs + - *build-and-create-float-ref-outputs - comp_args="--mld --ssnr --odg" @@ -418,10 +467,129 @@ stages: junit: - report-junit.xml -.ivas-pytest-on-merge-request-anchor: &ivas-pytest-on-merge-request-anchor - stage: test +.check-up-to-date-in-comparison-jobs: &check-up-to-date-in-comparison-jobs + - *get-commits-behind-count + - | + if [ $commits_behind_count -ne 0 ]; then + set +x + echo -e "Your branch is $commits_behind_count commits behind the target branch, possibly main changed during your pipeline run. Checking bitexactness or testing for regressions now can result in meaningless results. Run\n\t git pull origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME\nto update." + exit 1 + fi + +.check-be-to-target-anchor: &check-be-to-target-anchor + stage: check-be needs: ["build-codec-linux-make"] timeout: "300 minutes" + variables: + XML_REPORT: "report--$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.xml" + HTML_REPORT: "report--$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.html" + PYTEST_LOG_TARGET_BRANCH: "pytest-log-$CI_MERGE_REQUEST_TARGET_BRANCH_NAME.txt" + script: + - *print-common-info + + - set -euxo pipefail + + - *update-scripts-repo + - python3 tests/create_short_testvectors.py + + - if [ $USE_LTV -eq 1 ]; then + - *update-ltv-repo + - *copy-ltv-files-to-testv-dir + - testcase_timeout=$TESTCASE_TIMEOUT_LTV + - else + - testcase_timeout=$TESTCASE_TIMEOUT_STV + - fi + + - python3 ci/remove_unsupported_testcases.py $PRM_FILES + - python3 scripts/prepare_combined_format_inputs.py + + - if [ $LEVEL_SCALING != "1.0" ];then + - *apply-testv-scaling + - fi + + - *build-float-ref-binaries + - *build-merge-target-binaries + - make clean + - make -j + - *check-up-to-date-in-comparison-jobs + + - exit_code_target=0 + - python3 -m pytest $TEST_SUITE -v --update_ref 1 --create_ref -n auto --ref_encoder_path $MERGE_TARGET_ENCODER_PATH --ref_decoder_path $MERGE_TARGET_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH > $PYTEST_LOG_TARGET_BRANCH || exit_code_target=$? + + - exit_code=0 + - rm -rf .pytest_cache || true + - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --mld --ssnr --odg --ref_encoder_path $MERGE_TARGET_ENCODER_PATH --ref_decoder_path $MERGE_TARGET_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$? + + - if [ $exit_code -ne 0 ]; then + - exit_code=$EXIT_CODE_NON_BE + - zip -r $PYTEST_CACHE_ARTIFACT .pytest_cache + + - grep "^FAILED" pytest_log.txt | sed "s/^FAILED /'/" | sed "s/] - .*/]'/" | tr "\n" " " > $FAILED_TESTCASES_LIST || true + - grep "^FAILED" pytest_log.txt | sed "s/^FAILED //" | sed "s/] - .*/]/" > failed_testcases_for_printing.txt || true + - num_failures=$(wc -l < failed_testcases_for_printing.txt) + + - grep "^ERROR" pytest_log.txt | sed "s/^ERROR /'/" | sed "s/] - .*/]'/" | tr "\n" " " > $ERRORS_TESTCASES_LIST || true + - grep "^ERROR" pytest_log.txt | sed "s/^ERROR //" | sed "s/] - .*/]/" > errors_testcases_for_printing.txt || true + - num_errors=$(wc -l < errors_testcases_for_printing.txt) + + - *print-results-banner + - echo "Found these $num_failures non-bitexact testcases:" + - cat failed_testcases_for_printing.txt + + - echo "Reproduce locally with:" + - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $FLOAT_REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" + - echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH" + - echo "The individual command lines can be found in the html report in the job artifacts." + - if [ $num_errors -ne 0 ]; then + - exit_code=1 + - echo "There were errors present in the following testcases:" + - cat errors_testcases_for_printing.txt + - fi + + - exit $exit_code + - else + # create empty files to not have errors at artifact stage + - touch $FAILED_TESTCASES_LIST + - touch $ERRORS_TESTCASES_LIST + - touch $PYTEST_CACHE_ARTIFACT + - *print-results-banner + - echo "All testcases are bitexact." + - fi + - exit $exit_code + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" + expire_in: 1 week + when: always + paths: + - $XML_REPORT + - $HTML_REPORT + - $FAILED_TESTCASES_LIST + - $ERRORS_TESTCASES_LIST + - pytest_log.txt + - $PYTEST_CACHE_ARTIFACT + - $FLOAT_REF_COMMIT_FILE + - $CUT_COMMIT_FILE + - $MERGE_TARGET_COMMIT_FILE + - $PYTEST_LOG_TARGET_BRANCH + expose_as: "pytest compare results" + reports: + junit: + - $XML_REPORT + - $XML_REPORT + +.overwrite-pytest-cache-with-artifact: &overwrite-pytest-cache-with-artifact + - if [ -f $PYTEST_CACHE_ARTIFACT ]; then + - rm -rf .pytest_cache || true + - unzip $PYTEST_CACHE_ARTIFACT + - fi + +.check-regressions-pytest-anchor: &check-regressions-pytest-anchor + stage: test + timeout: "300 minutes" variables: XML_REPORT_BRANCH: "report-junit-branch-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.xml" XML_REPORT_MAIN: "report-junit-main-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.xml" @@ -432,9 +600,22 @@ stages: IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME" SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html" script: - - set -euxo pipefail - *print-common-info + + - set -euxo pipefail + + - if [ -s $FAILED_TESTCASES_LIST ]; then + - *overwrite-pytest-cache-with-artifact + - export PYTEST_ADDOPTS=--last-failed + - else + # turn off echoing back of commands for result printout + - *print-results-banner + - echo -e "All tested cases were bit-exact between $CI_MERGE_REQUEST_TARGET_BRANCH_NAME and $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME.\nNo need to check for regressions. All is fine." + - exit 0 + - fi + - *update-scripts-repo + - if [ $USE_LTV -eq 1 ]; then - *update-ltv-repo - *copy-ltv-files-to-testv-dir @@ -454,9 +635,11 @@ stages: ### run branch first # this per default builds the branch and the reference and creates the reference outputs - - *build-and-create-reference-outputs - - exit_code=0 - - python3 -m pytest --tb=no $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || exit_code=$? + - *build-and-create-float-ref-outputs + - *check-up-to-date-in-comparison-jobs + # need to restore cache again + - *overwrite-pytest-cache-with-artifact + - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true - zero_errors_branch=$(cat $XML_REPORT_BRANCH | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py $XML_REPORT_BRANCH $CSV_BRANCH @@ -473,44 +656,37 @@ stages: - git pull - make clean - make -j - - python3 -m pytest --tb=no $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true + # need to restore cache again + - *overwrite-pytest-cache-with-artifact + - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true - python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN - # If outputs of main and branch are equal, have equal reports and no run errors were encountered, the job will pass. - - diff_sba=0 - - diff_param=0 - - diff_report=0 - # SHORT_TEST_SUITE_ENCODER does not contain test_sba.py. This leads to non-existing output folders being compared and to diff_sba=1. Therefore, this is skipped for the encoder tests - - if [ "$TEST_SUITE" != "$SHORT_TEST_SUITE_ENCODER" ]; then - - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/sba_bs/raw tests/dut_branch/sba_bs/raw || diff_sba=$? - - fi - - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/param_file/dec tests/dut_branch/param_file/dec || diff_param=$? - - diff $CSV_BRANCH $CSV_MAIN || diff_report=$? - - if [ $diff_param -eq 0 ] && [ $diff_sba -eq 0 ] && [ $diff_report -eq 0 ] && [ $zero_errors_branch -eq 1 ]; then - - echo "Output BE to main, identical report and no run errors encountered." - # Add dummy files to avoid warning on missing artifacts - - touch changes_crashes.csv - - touch changes_MLD.csv - - touch changes_MAXIMUM_ABS_DIFF.csv - - touch changes_MIN_SSNR.csv - - touch changes_MIN_ODG.csv - - exit 0; - - fi - ### compare the two csv files for regressions - regressions_found=0 - - python3 scripts/basop_check_for_changes_in_testcases.py --xml_report $XML_REPORT_BRANCH $CSV_BRANCH $CSV_MAIN || regressions_found=$? + - python3 scripts/basop_check_for_changes_in_testcases.py --show_improvements --xml_report $XML_REPORT_BRANCH $CSV_BRANCH $CSV_MAIN > regression_log.txt || regressions_found=$? - - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit_code=$EXIT_CODE_NON_BE; fi - - if [ $zero_errors_branch != 1 ]; then echo "Run errors encountered!"; exit_code=$EXIT_CODE_NON_BE; fi - - if [ $regressions_found != 0 ] && [ "$SKIP_REGRESSION_CHECK" != "true" ]; then + - exit_code=0 + - *print-results-banner + - if [ $zero_errors_branch != 1 ]; then + - echo "Run errors encountered!" + - exit_code=$EXIT_CODE_FAIL + - echo "Reproduce locally with:" + - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $FLOAT_REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $ERRORS_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" + - echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $ERRORS_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH" + - echo "The individual command lines can be found in the changes*.csv files in the job artifacts." + - elif [ $regressions_found != 0 ] && [ "$SKIP_REGRESSION_CHECK" != "true" ]; then + - cat regression_log.txt - if [ $allow_regressions_flag == 0 ]; then - - echo "Detected regression wrt to main, [allow regression] not set!" + - echo "Detected regression wrt to $CI_MERGE_REQUEST_TARGET_BRANCH_NAME, [allow regression] not set!" - exit_code=$EXIT_CODE_FAIL; - else - - echo "Detected regression wrt to main, [allow regression] set." + - echo "Detected regression wrt to $CI_MERGE_REQUEST_TARGET_BRANCH_NAME, [allow regression] set." - exit_code=$EXIT_CODE_NON_BE; - fi + - echo "Reproduce locally with:" + - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $FLOAT_REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" + - echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH" + - echo "The individual command lines can be found in the changes*.csv files in the job artifacts." - fi - exit $exit_code @@ -532,6 +708,9 @@ stages: - $CSV_MAIN - $SUMMARY_HTML_ARTIFACT_NAME - $IMAGES_ARTIFACT_NAME + - $FLOAT_REF_COMMIT_FILE + - $CUT_COMMIT_FILE + - $MERGE_TARGET_COMMIT_FILE - changes_crashes.csv - changes_MLD.csv - changes_MAXIMUM_ABS_DIFF.csv @@ -561,7 +740,7 @@ stages: - *copy-ltv-files-to-testv-dir - python3 ci/remove_unsupported_testcases.py $PRM_FILES - - *build-reference-binaries + - *build-float-ref-binaries - set -euxo pipefail - make_args="CLANG=$CLANG_NUM" - if [[ $CLANG_NUM == 3 ]]; then @@ -606,7 +785,7 @@ stages: - if [ $LEVEL_SCALING != "1.0" ];then - *apply-testv-scaling - fi - - *build-reference-and-dut-binaries + - *build-float-ref-and-dut-binaries ### run pytest - exit_code=0 @@ -674,6 +853,35 @@ uninterruptible: # verification jobs # --------------------------------------------------------------- +branch-is-up-to-date-with-target-pre: + extends: + - .rules-merge-request + stage: prevalidate + needs: [] + tags: + - ivas-basop-linux + script: + - *get-commits-behind-count + - | + if [ $commits_behind_count -ne 0 ]; then + echo -e "Your branch is $commits_behind_count commits behind the target branch, run\n\tgit pull origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME\nto update." + exit 1 + fi + +branch-is-up-to-date-with-target-post: + extends: + - .rules-merge-request + stage: postvalidate + tags: + - ivas-basop-linux + script: + - *get-commits-behind-count + - | + if [ $commits_behind_count -ne 0 ]; then + echo -e "Your branch is $commits_behind_count commits behind the target branch, possibly main changed during your pipeline run, run\n\tgit pull origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME\nto update." + exit 1 + fi + clang-format-check: extends: - .test-job-linux @@ -804,69 +1012,155 @@ build-codec-windows-msbuild: # Short test jobs that run in merge request pipelines # --------------------------------------------------------------- -### jobs that test fx encoder -> flt decoder -ivas-pytest-compare_to_main-short-enc: +### jobs that check for bitexactness of fx encoder and decoder +check-be-to-target-short-enc-0db: extends: - .rules-pytest-to-main-short - .test-job-linux before_script: - USE_LTV=0 - DUT_DECODER_PATH=./IVAS_dec_ref + - MERGE_TARGET_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - LEVEL_SCALING=1.0 - - SKIP_REGRESSION_CHECK="true" - <<: *ivas-pytest-on-merge-request-anchor + - rm -rf tests/dut tests/ref + <<: *check-be-to-target-anchor -ivas-pytest-compare_to_main-short-enc-lev-10: +check-be-to-target-short-enc-+10db: extends: - .rules-pytest-to-main-short - .test-job-linux before_script: - USE_LTV=0 - DUT_DECODER_PATH=./IVAS_dec_ref + - MERGE_TARGET_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - - LEVEL_SCALING=0.3162 - - SKIP_REGRESSION_CHECK="true" - <<: *ivas-pytest-on-merge-request-anchor + - LEVEL_SCALING=3.162 + - rm -rf tests/dut tests/ref + <<: *check-be-to-target-anchor -ivas-pytest-compare_to_main-short-enc-lev+10: +check-be-to-target-short-enc--10db: extends: - .rules-pytest-to-main-short - .test-job-linux before_script: - USE_LTV=0 - DUT_DECODER_PATH=./IVAS_dec_ref + - MERGE_TARGET_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - - LEVEL_SCALING=3.162 - - SKIP_REGRESSION_CHECK="true" - <<: *ivas-pytest-on-merge-request-anchor + - LEVEL_SCALING=0.3162 + - rm -rf tests/dut tests/ref + <<: *check-be-to-target-anchor -### jobs that test flt encoder -> fx decoder -ivas-pytest-compare_to_main-short-dec: +check-be-to-target-short-dec-0db: extends: - .rules-pytest-to-main-short - .test-job-linux before_script: - USE_LTV=0 - DUT_ENCODER_PATH=./IVAS_cod_ref + - MERGE_TARGET_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=1.0 - rm -rf tests/dut tests/ref - <<: *ivas-pytest-on-merge-request-anchor + <<: *check-be-to-target-anchor -ivas-pytest-compare_to_main-short-dec-lev-10: +check-be-to-target-short-dec-+10db: extends: - .rules-pytest-to-main-short - .test-job-linux before_script: - USE_LTV=0 - DUT_ENCODER_PATH=./IVAS_cod_ref + - MERGE_TARGET_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=3.162 + - rm -rf tests/dut tests/ref + <<: *check-be-to-target-anchor + +check-be-to-target-short-dec--10db: + extends: + - .rules-pytest-to-main-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_ENCODER_PATH=./IVAS_cod_ref + - MERGE_TARGET_ENCODER_PATH=./IVAS_cod_ref + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=0.3162 + - rm -rf tests/dut tests/ref + <<: *check-be-to-target-anchor + +### jobs that check for regressions on non-BE testcases +check-regressions-short-enc-0db: + stage: test + needs: + - job: "check-be-to-target-short-enc-0db" + artifacts: true + extends: + - .rules-pytest-to-main-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_DECODER_PATH=./IVAS_dec_ref + - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" + - LEVEL_SCALING=1.0 + - rm -rf tests/dut tests/ref + <<: *check-regressions-pytest-anchor + +check-regressions-short-enc-+10db: + stage: test + needs: + - job: "check-be-to-target-short-enc-+10db" + artifacts: true + extends: + - .rules-pytest-to-main-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_DECODER_PATH=./IVAS_dec_ref + - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" + - LEVEL_SCALING=3.162 + - rm -rf tests/dut tests/ref + <<: *check-regressions-pytest-anchor + +check-regressions-short-enc--10db: + stage: test + needs: + - job: "check-be-to-target-short-enc--10db" + artifacts: true + extends: + - .rules-pytest-to-main-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_DECODER_PATH=./IVAS_dec_ref + - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - LEVEL_SCALING=0.3162 - rm -rf tests/dut tests/ref - <<: *ivas-pytest-on-merge-request-anchor + <<: *check-regressions-pytest-anchor -ivas-pytest-compare_to_main-short-dec-lev+10: +check-regressions-short-dec-0db: + stage: test + needs: + - job: "check-be-to-target-short-dec-0db" + artifacts: true + extends: + - .rules-pytest-to-main-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_ENCODER_PATH=./IVAS_cod_ref + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=1.0 + - rm -rf tests/dut tests/ref + <<: *check-regressions-pytest-anchor + +check-regressions-short-dec-+10db: + stage: test + needs: + - job: "check-be-to-target-short-dec-+10db" + artifacts: true extends: - .rules-pytest-to-main-short - .test-job-linux @@ -876,7 +1170,23 @@ ivas-pytest-compare_to_main-short-dec-lev+10: - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=3.162 - rm -rf tests/dut tests/ref - <<: *ivas-pytest-on-merge-request-anchor + <<: *check-regressions-pytest-anchor + +check-regressions-short-dec--10db: + stage: test + needs: + - job: "check-be-to-target-short-dec--10db" + artifacts: true + extends: + - .rules-pytest-to-main-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_ENCODER_PATH=./IVAS_cod_ref + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=0.3162 + - rm -rf tests/dut tests/ref + <<: *check-regressions-pytest-anchor # --------------------------------------------------------------- # Short test jobs for running from web interface or schedule @@ -1207,7 +1517,7 @@ coverage-test-on-main-scheduled: - *update-scripts-repo - *update-ltv-repo - *copy-ltv-files-to-testv-dir - - *build-reference-binaries + - *build-float-ref-binaries # Build DuT binaries with GCOV - make clean - make GCOV=1 -j @@ -1418,7 +1728,7 @@ voip-be-on-merge-request: - *print-common-info - *update-scripts-repo - *update-ltv-repo - - *build-reference-and-dut-binaries + - *build-float-ref-and-dut-binaries - *complexity-measurements-setup - which coan artifacts: