From caffabf922064e3062536ff3e496daff940e351b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 28 Jan 2025 16:20:25 +0100 Subject: [PATCH 1/5] make the encoder MR pipeline tests compare to main as well --- .gitlab-ci.yml | 89 +++++++++++++++++++++----------------------------- 1 file changed, 37 insertions(+), 52 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f62acb284..768f12e64 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -211,17 +211,6 @@ stages: - if: $CI_PIPELINE_SOURCE == 'schedule' when: never -# TODO: only temporary as long the MR encoder tests should not compare to main -.rules-pytest-to-ref-enc-short-temp: - rules: - - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare" - - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main - - if: $CI_PIPELINE_SOURCE == 'push' - when: never - - if: $CI_PIPELINE_SOURCE == 'schedule' - when: never - .rules-pytest-to-main-short: rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main @@ -727,39 +716,38 @@ build-codec-linux-debugging-make: # --------------------------------------------------------------- ### jobs that test fx encoder -> flt decoder -# TODO: reenable once encoder tests shall compare to main -# ivas-pytest-compare_to_main-short-enc: -# extends: -# - .rules-pytest-to-main-short -# - .test-job-linux -# before_script: -# - USE_LTV=0 -# - DUT_DECODER_PATH=./IVAS_dec_ref -# - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" -# - LEVEL_SCALING=1.0 -# <<: *ivas-pytest-on-merge-request-anchor - -# ivas-pytest-compare_to_main-short-enc-lev-10: -# extends: -# - .rules-pytest-to-main-short -# - .test-job-linux -# before_script: -# - USE_LTV=0 -# - DUT_DECODER_PATH=./IVAS_dec_ref -# - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" -# - LEVEL_SCALING=0.3162 -# <<: *ivas-pytest-on-merge-request-anchor - -# ivas-pytest-compare_to_main-short-enc-lev+10: -# extends: -# - .rules-pytest-to-main-short -# - .test-job-linux -# before_script: -# - USE_LTV=0 -# - DUT_DECODER_PATH=./IVAS_dec_ref -# - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" -# - LEVEL_SCALING=3.162 -# <<: *ivas-pytest-on-merge-request-anchor +ivas-pytest-compare_to_main-short-enc: + extends: + - .rules-pytest-to-main-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_DECODER_PATH=./IVAS_dec_ref + - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" + - LEVEL_SCALING=1.0 + <<: *ivas-pytest-on-merge-request-anchor + +ivas-pytest-compare_to_main-short-enc-lev-10: + extends: + - .rules-pytest-to-main-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_DECODER_PATH=./IVAS_dec_ref + - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" + - LEVEL_SCALING=0.3162 + <<: *ivas-pytest-on-merge-request-anchor + +ivas-pytest-compare_to_main-short-enc-lev+10: + extends: + - .rules-pytest-to-main-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_DECODER_PATH=./IVAS_dec_ref + - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" + - LEVEL_SCALING=3.162 + <<: *ivas-pytest-on-merge-request-anchor ### jobs that test flt encoder -> fx decoder ivas-pytest-compare_to_main-short-dec: @@ -805,8 +793,7 @@ ivas-pytest-compare_to_main-short-dec-lev+10: ### jobs that test fx encoder -> flt decoder ivas-pytest-compare_to_ref-short-enc: extends: - #- .rules-pytest-to-ref-short - - .rules-pytest-to-ref-enc-short-temp + - .rules-pytest-to-ref-short - .test-job-linux before_script: - USE_LTV=0 @@ -818,9 +805,8 @@ ivas-pytest-compare_to_ref-short-enc: ivas-pytest-compare_to_ref-short-enc-lev-10: extends: - #- .rules-pytest-to-ref-short - - .rules-pytest-to-ref-enc-short-temp - - .test-job-linux + - .rules-pytest-to-ref-short + - .test-job-linux before_script: - USE_LTV=0 - ENCODER_TEST="true" @@ -831,9 +817,8 @@ ivas-pytest-compare_to_ref-short-enc-lev-10: ivas-pytest-compare_to_ref-short-enc-lev+10: extends: - #- .rules-pytest-to-ref-short - - .rules-pytest-to-ref-enc-short-temp - - .test-job-linux + - .rules-pytest-to-ref-short + - .test-job-linux before_script: - USE_LTV=0 - ENCODER_TEST="true" -- GitLab From 1e8532077a1c69edd828233a11df3f7b7f9442f6 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 28 Jan 2025 16:58:29 +0100 Subject: [PATCH 2/5] skip regressions check for encoder tests this way, the tests will return the allowed-to-fail exit code if non-BE is detected and 0 if everything is BE --- .gitlab-ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 768f12e64..fdf64eef3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -439,7 +439,7 @@ stages: - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit_code=$EXIT_CODE_NON_BE; fi - if [ $zero_errors_branch != 1 ]; then echo "Run errors encountered!"; exit_code=$EXIT_CODE_NON_BE; fi - - if [ $regressions_found != 0 ]; then + - if [ $regressions_found != 0 && "$SKIP_REGRESSION_CHECK" != "true" ]; then - if [ $allow_regressions_flag == 0 ]; then - echo "Detected regression wrt to main, [allow regression] not set!" - exit_code=$EXIT_CODE_FAIL; @@ -725,6 +725,7 @@ ivas-pytest-compare_to_main-short-enc: - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - LEVEL_SCALING=1.0 + - SKIP_REGRESSION_CHECK="true" <<: *ivas-pytest-on-merge-request-anchor ivas-pytest-compare_to_main-short-enc-lev-10: @@ -736,6 +737,7 @@ ivas-pytest-compare_to_main-short-enc-lev-10: - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - LEVEL_SCALING=0.3162 + - SKIP_REGRESSION_CHECK="true" <<: *ivas-pytest-on-merge-request-anchor ivas-pytest-compare_to_main-short-enc-lev+10: @@ -747,6 +749,7 @@ ivas-pytest-compare_to_main-short-enc-lev+10: - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - LEVEL_SCALING=3.162 + - SKIP_REGRESSION_CHECK="true" <<: *ivas-pytest-on-merge-request-anchor ### jobs that test flt encoder -> fx decoder -- GitLab From b3d6e78f00f3e7e637ca831a8dae12365f02aa0b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 28 Jan 2025 18:27:04 +0100 Subject: [PATCH 3/5] fix if command --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fdf64eef3..accb790ba 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -439,7 +439,7 @@ stages: - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit_code=$EXIT_CODE_NON_BE; fi - if [ $zero_errors_branch != 1 ]; then echo "Run errors encountered!"; exit_code=$EXIT_CODE_NON_BE; fi - - if [ $regressions_found != 0 && "$SKIP_REGRESSION_CHECK" != "true" ]; then + - if [ $regressions_found != 0 ] && [ "$SKIP_REGRESSION_CHECK" != "true" ]; then - if [ $allow_regressions_flag == 0 ]; then - echo "Detected regression wrt to main, [allow regression] not set!" - exit_code=$EXIT_CODE_FAIL; -- GitLab From a8c085f8630414c14845ff0bd153bba199949b4f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 28 Jan 2025 18:48:02 +0100 Subject: [PATCH 4/5] set bash echo + better debugging options --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index accb790ba..dfd8e9c5a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -372,6 +372,7 @@ stages: IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME" SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html" script: + - set -euxo pipefail - *print-common-info - *update-scripts-repo - if [ $USE_LTV -eq 1 ]; then -- GitLab From 855096d98c82a28de2f1b1ba970537480058409c Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 28 Jan 2025 18:56:48 +0100 Subject: [PATCH 5/5] skip diff_sba part for encoder tests --- .gitlab-ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dfd8e9c5a..33092e163 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -420,7 +420,10 @@ stages: - diff_sba=0 - diff_param=0 - diff_report=0 - - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/sba_bs/raw tests/dut_branch/sba_bs/raw || diff_sba=$? + # SHORT_TEST_SUITE_ENCODER does not contain test_sba.py. This leads to non-existing output folders being compared and to diff_sba=1. Therefore, this is skipped for the encoder tests + - if [ "$TEST_SUITE" != "$SHORT_TEST_SUITE_ENCODER" ]; then + - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/sba_bs/raw tests/dut_branch/sba_bs/raw || diff_sba=$? + - fi - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/param_file/dec tests/dut_branch/param_file/dec || diff_param=$? - diff $CSV_BRANCH $CSV_MAIN || diff_report=$? - if [ $diff_param -eq 0 ] && [ $diff_sba -eq 0 ] && [ $diff_report -eq 0 ] && [ $zero_errors_branch -eq 1 ]; then -- GitLab