Merge branch 'ci/fix-exit-codes' into 'main'

    - if [ $zero_errors != 1 ]; then

    -     echo "Run errors encountered!"

    # TODO: temporary only to not fail pipelines on crashes in encoder

    -     if [[ "$CI_JOB_NAME" =~ "ivas-pytest-compare_to_ref-short-enc" ]]; then

    # TODO: temporary only to not fail MR pipelines on crashes

    -     if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then

    -         exit $EXIT_CODE_NON_BE

    -     else

    -         exit $EXIT_CODE_FAIL

    # this per default builds the branch and the reference and creates the reference outputs

    - *build-and-create-reference-outputs

    - exit_code=0

    - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || exit_code=$?

    - python3 -m pytest --tb=no $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || exit_code=$?

    - zero_errors_branch=$(cat $XML_REPORT_BRANCH | grep -c 'errors="0"') || true

    - python3 scripts/parse_xml_report.py $XML_REPORT_BRANCH $CSV_BRANCH

    # Store branch outputs for comparison

    - mv tests/dut tests/dut_branch

    # create the summary based on the branch

    - mkdir $IMAGES_ARTIFACT_NAME

    - for MEASURE in MLD DIFF SSNR;do python3 scripts/create_histogram_summary.py $CSV_BRANCH $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE; done

    - git pull

    - make clean

    - make -j

    - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true

    - python3 -m pytest --tb=no $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true

    - python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN

    # If outputs of main and branch are equal, have equal reports and no run errors were encountered, the job will pass.

    - diff_sba=0

    - diff_param=0

    - diff_report=0

    - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/sba_bs/raw tests/dut_branch/sba_bs/raw || diff_sba=$?

    - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/param_file/dec tests/dut_branch/param_file/dec || diff_param=$?

    - diff $CSV_BRANCH $CSV_MAIN || diff_report=$?

    - if [ $diff_param -eq 0 ] && [ $diff_sba -eq 0 ] && [ $diff_report -eq 0 ] && [ $zero_errors_branch -eq 1 ]; then

    -    echo "Output BE to main, identical report and no run errors encountered."

    -    exit 0;

    - fi

    ### compare the two csv files for regressions

    - regressions_found=0

    - python3 scripts/basop_check_for_changes_in_testcases.py $CSV_BRANCH $CSV_MAIN || regressions_found=$?

    - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit_code=$EXIT_CODE_NON_BE; fi

    - if [ $zero_errors_branch != 1 ]; then echo "Run errors encountered!"; exit_code=$EXIT_CODE_NON_BE; fi

    - if [ $regressions_found != 0 ]; then

    -    echo "Detected regression wrt to main!"

    -    if [ $allow_regressions_flag == 0 ]; then

    -        exit $EXIT_CODE_FAIL;

    -        echo "Detected regression wrt to main, [allow regression] not set!"

    -        exit_code=$EXIT_CODE_FAIL;

    -    else

    -        exit $EXIT_CODE_NON_BE;

    -        echo "Detected regression wrt to main, [allow regression] set."

    -        exit_code=$EXIT_CODE_NON_BE;

    -    fi

    - fi

    - if [ $zero_errors_branch != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_NON_BE; fi

    - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi

    - exit 0

    - exit $exit_code

  after_script:

    - rm -rf tests/dut tests/ref

  allow_failure:

    - LEVEL_SCALING=1.0

  <<: *ivas-pytest-anchor

ivas-pytest-compare_to_ref-short-lev-10:

ivas-pytest-compare_to_ref-short-dec-lev-10:

  extends:

    - .rules-pytest-to-ref-short

    - .test-job-linux