Commit dbb55697 authored by Jan Kiene's avatar Jan Kiene
Browse files

improve reporting in be check job

parent 4fcc08c1

.gitlab-ci.yml

+40 −29
Original line number Diff line number Diff line
@@ -35,6 +35,7 @@ variables: 
  COMPARE_DMX: ""

  SKIP_REGRESSION_CHECK: ""

  FAILED_TESTCASES_LIST: "failed-testcases.txt"

  ERRORS_TESTCASES_LIST: "errors-testcases.txt"

  PYTEST_CACHE_ARTIFACT: "pytest_cache.zip"

  MANUAL_PIPELINE_TYPE:

    description: "Type for the manual pipeline run. Use 'pytest-compare' to run comparison test against reference float codec."
@@ -305,6 +306,14 @@ stages: 
  tags:

    - ivas-windows



.print-results-banner: &print-results-banner

  -   set +x

  -   echo ""

  -   echo "=================================================================================================================="

  -   echo "================================================== TEST RESULTS =================================================="

  -   echo "=================================================================================================================="

  -   echo ""



# template for test jobs on linux that need the TESTV_DIR

.test-job-linux-needs-testv-dir:

  extends: .test-job-linux
@@ -444,17 +453,37 @@ stages: 
    - exit_code=0

    - rm -rf .pytest_cache || true

    - python3 -m pytest -k "stereo and at" --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$?

    - zero_errors_branch=$(cat $XML_REPORT | grep -c 'errors="0"') || true



    - touch $FAILED_TESTCASES_LIST

    - touch $ERRORS_TESTCASES_LIST

    - touch $PYTEST_CACHE_ARTIFACT

    - if [ $exit_code -ne 0 ]; then

    -   exit_code=$EXIT_CODE_NON_BE

    -   zip -r $PYTEST_CACHE_ARTIFACT .pytest_cache



    -   grep "^FAILED" pytest_log.txt | sed 's/^FAILED //' | sed 's/] - .*/]/' | tr "\n" " " > $FAILED_TESTCASES_LIST

    - fi

    -   grep "^FAILED" pytest_log.txt | sed 's/^FAILED //' | sed 's/] - .*/]/' > failed_testcases_for_printing.txt

    -   num_failures=$(wc -l < failed_testcases_for_printing.txt)



    - cat $FAILED_TESTCASES_LIST

    -   grep "^ERROR" pytest_log.txt | sed 's/^ERROR //' | sed 's/] - .*/]/' | tr "\n" " " > $ERRORS_TESTCASES_LIST

    -   grep "^ERROR" pytest_log.txt | sed 's/^ERROR //' | sed 's/] - .*/]/' > errors_testcases_for_printing.txt

    -   num_errors=$(wc -l < errors_testcases_for_printing.txt)



    -   *print-results-banner

    -   echo "Found these $num_failures non-bitexact testcases:"

    -   cat failed_testcases_for_printing.txt



    -   if [ $num_errors -ne 0 ]; then

    -     exit_code=1

    -     echo "There were errors present in the following testcases:"

    -     cat errors_testcases_for_printing.txt

    -   fi



    - zip -r $PYTEST_CACHE_ARTIFACT .pytest_cache

    -   exit $exit_code

    - else

    -   *print-results-banner

    -   echo "All testcases are bitexact."

    - fi

    - exit $exit_code



  allow_failure:
@@ -502,7 +531,10 @@ stages: 
    -   *overwrite-pytest-cache-with-artifact

    -   export PYTEST_ADDOPTS=--last-failed

    - else

    -   echo "All tested cases were bit-exact between $CI_MERGE_REQUEST_TARGET_BRANCH_NAME and $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME. No need to check for regressions."

    # turn off echoing back of commands for result printout

    -   *print-results-banner

    -   echo "All tested cases were bit-exact between $CI_MERGE_REQUEST_TARGET_BRANCH_NAME and $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME."

    -   echo "No need to check for regressions. All is fine."

    -   exit 0

    - fi
@@ -528,10 +560,9 @@ stages: 
    ### run branch first

    # this per default builds the branch and the reference and creates the reference outputs

    - *build-and-create-reference-outputs

    - exit_code=0

    # need to restore cache again

    - *overwrite-pytest-cache-with-artifact

    - python3 -m pytest --tb=no $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || exit_code=$?

    - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true

    - zero_errors_branch=$(cat $XML_REPORT_BRANCH | grep -c 'errors="0"') || true

    - python3 scripts/parse_xml_report.py $XML_REPORT_BRANCH $CSV_BRANCH
@@ -550,34 +581,14 @@ stages: 
    - make -j

    # need to restore cache again

    - *overwrite-pytest-cache-with-artifact

    - python3 -m pytest --tb=no $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true

    - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true

    - python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN



    # If outputs of main and branch are equal, have equal reports and no run errors were encountered, the job will pass.

    - diff_sba=0

    - diff_param=0

    - diff_report=0

    # SHORT_TEST_SUITE_ENCODER does not contain test_sba.py. This leads to non-existing output folders being compared and to diff_sba=1. Therefore, this is skipped for the encoder tests

    - if [ "$TEST_SUITE" != "$SHORT_TEST_SUITE_ENCODER" ]; then

    -     python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/sba_bs/raw tests/dut_branch/sba_bs/raw || diff_sba=$?

    - fi

    - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/param_file/dec tests/dut_branch/param_file/dec || diff_param=$?

    - diff $CSV_BRANCH $CSV_MAIN || diff_report=$?

    - if [ $diff_param -eq 0 ] && [ $diff_sba -eq 0 ] && [ $diff_report -eq 0 ] && [ $zero_errors_branch -eq 1 ]; then

    -    echo "Output BE to main, identical report and no run errors encountered."

         # Add dummy files to avoid warning on missing artifacts

    -    touch changes_crashes.csv

    -    touch changes_MLD.csv

    -    touch changes_MAXIMUM_ABS_DIFF.csv

    -    touch changes_MIN_SSNR.csv

    -    touch changes_MIN_ODG.csv

    -    exit 0;

    - fi



    ### compare the two csv files for regressions

    - regressions_found=0

    - python3 scripts/basop_check_for_changes_in_testcases.py --xml_report $XML_REPORT_BRANCH $CSV_MAIN $CSV_BRANCH || regressions_found=$?



    - exit_code=0

    - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit_code=$EXIT_CODE_NON_BE; fi

    - if [ $zero_errors_branch != 1 ]; then echo "Run errors encountered!"; exit_code=$EXIT_CODE_NON_BE; fi

    - if [ $regressions_found != 0 ] && [ "$SKIP_REGRESSION_CHECK" != "true" ]; then