Commit c374ed75 authored by Sandesh Venkatesh's avatar Sandesh Venkatesh
Browse files

Merge branch '1233-ci-enable-testing-for-regressions-in-the-compare_to_enc-tests' into 'main' 

[CI] enable regressions check for "compare_to_enc" tests on main

Closes #1233

See merge request !1039
parents e3c8197d 855096d9

.gitlab-ci.yml

+46 −54
Original line number Diff line number Diff line
@@ -211,17 +211,6 @@ stages: 
    - if: $CI_PIPELINE_SOURCE == 'schedule'

      when: never



# TODO: only temporary as long the MR encoder tests should not compare to main

.rules-pytest-to-ref-enc-short-temp:

  rules:

    - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare"

    - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main

    - if: $CI_PIPELINE_SOURCE == 'push'

      when: never

    - if: $CI_PIPELINE_SOURCE == 'schedule'

      when: never



.rules-pytest-to-main-short:

  rules:

    - if: $CI_PIPELINE_SOURCE == 'merge_request_event' && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main" # only have MR pipelines for MRs to main
@@ -383,6 +372,7 @@ stages: 
    IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME"

    SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html"

  script:

    - set -euxo pipefail

    - *print-common-info

    - *update-scripts-repo

    - if [ $USE_LTV -eq 1 ]; then
@@ -430,7 +420,10 @@ stages: 
    - diff_sba=0

    - diff_param=0

    - diff_report=0

    # SHORT_TEST_SUITE_ENCODER does not contain test_sba.py. This leads to non-existing output folders being compared and to diff_sba=1. Therefore, this is skipped for the encoder tests

    - if [ "$TEST_SUITE" != "$SHORT_TEST_SUITE_ENCODER" ]; then

    -     python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/sba_bs/raw tests/dut_branch/sba_bs/raw || diff_sba=$?

    - fi

    - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/param_file/dec tests/dut_branch/param_file/dec || diff_param=$?

    - diff $CSV_BRANCH $CSV_MAIN || diff_report=$?

    - if [ $diff_param -eq 0 ] && [ $diff_sba -eq 0 ] && [ $diff_report -eq 0 ] && [ $zero_errors_branch -eq 1 ]; then
@@ -450,7 +443,7 @@ stages: 


    - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit_code=$EXIT_CODE_NON_BE; fi

    - if [ $zero_errors_branch != 1 ]; then echo "Run errors encountered!"; exit_code=$EXIT_CODE_NON_BE; fi

    - if [ $regressions_found != 0 ]; then

    - if [ $regressions_found != 0 ] && [ "$SKIP_REGRESSION_CHECK" != "true" ]; then

    -    if [ $allow_regressions_flag == 0 ]; then

    -        echo "Detected regression wrt to main, [allow regression] not set!"

    -        exit_code=$EXIT_CODE_FAIL;
@@ -727,39 +720,41 @@ build-codec-linux-debugging-make: 
# ---------------------------------------------------------------



### jobs that test fx encoder -> flt decoder

# TODO: reenable once encoder tests shall compare to main

# ivas-pytest-compare_to_main-short-enc:

#   extends:

#     - .rules-pytest-to-main-short

#     - .test-job-linux

#   before_script:

#     - USE_LTV=0

#     - DUT_DECODER_PATH=./IVAS_dec_ref

#     - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER"

#     - LEVEL_SCALING=1.0

#   <<: *ivas-pytest-on-merge-request-anchor



# ivas-pytest-compare_to_main-short-enc-lev-10:

#   extends:

#     - .rules-pytest-to-main-short

#     - .test-job-linux

#   before_script:

#     - USE_LTV=0

#     - DUT_DECODER_PATH=./IVAS_dec_ref

#     - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER"

#     - LEVEL_SCALING=0.3162

#   <<: *ivas-pytest-on-merge-request-anchor



# ivas-pytest-compare_to_main-short-enc-lev+10:

#   extends:

#     - .rules-pytest-to-main-short

#     - .test-job-linux

#   before_script:

#     - USE_LTV=0

#     - DUT_DECODER_PATH=./IVAS_dec_ref

#     - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER"

#     - LEVEL_SCALING=3.162

#   <<: *ivas-pytest-on-merge-request-anchor

ivas-pytest-compare_to_main-short-enc:

  extends:

    - .rules-pytest-to-main-short

    - .test-job-linux

  before_script:

    - USE_LTV=0

    - DUT_DECODER_PATH=./IVAS_dec_ref

    - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER"

    - LEVEL_SCALING=1.0

    - SKIP_REGRESSION_CHECK="true"

  <<: *ivas-pytest-on-merge-request-anchor



ivas-pytest-compare_to_main-short-enc-lev-10:

  extends:

    - .rules-pytest-to-main-short

    - .test-job-linux

  before_script:

    - USE_LTV=0

    - DUT_DECODER_PATH=./IVAS_dec_ref

    - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER"

    - LEVEL_SCALING=0.3162

    - SKIP_REGRESSION_CHECK="true"

  <<: *ivas-pytest-on-merge-request-anchor



ivas-pytest-compare_to_main-short-enc-lev+10:

  extends:

    - .rules-pytest-to-main-short

    - .test-job-linux

  before_script:

    - USE_LTV=0

    - DUT_DECODER_PATH=./IVAS_dec_ref

    - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER"

    - LEVEL_SCALING=3.162

    - SKIP_REGRESSION_CHECK="true"

  <<: *ivas-pytest-on-merge-request-anchor



### jobs that test flt encoder -> fx decoder

ivas-pytest-compare_to_main-short-dec:
@@ -805,8 +800,7 @@ ivas-pytest-compare_to_main-short-dec-lev+10: 
### jobs that test fx encoder -> flt decoder

ivas-pytest-compare_to_ref-short-enc:

  extends:

    #- .rules-pytest-to-ref-short

    - .rules-pytest-to-ref-enc-short-temp

    - .rules-pytest-to-ref-short

    - .test-job-linux

  before_script:

    - USE_LTV=0
@@ -818,8 +812,7 @@ ivas-pytest-compare_to_ref-short-enc: 


ivas-pytest-compare_to_ref-short-enc-lev-10:

  extends:

    #- .rules-pytest-to-ref-short

    - .rules-pytest-to-ref-enc-short-temp

    - .rules-pytest-to-ref-short

    - .test-job-linux

  before_script:

    - USE_LTV=0
@@ -831,8 +824,7 @@ ivas-pytest-compare_to_ref-short-enc-lev-10: 


ivas-pytest-compare_to_ref-short-enc-lev+10:

  extends:

    #- .rules-pytest-to-ref-short

    - .rules-pytest-to-ref-enc-short-temp

    - .rules-pytest-to-ref-short

    - .test-job-linux

  before_script:

    - USE_LTV=0