From bc9bec20b2ec08885df071361415dc324fd908b8 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Tue, 10 Sep 2024 17:47:18 +0200 Subject: [PATCH 01/14] change exit code handling to not mask crashes --- .gitlab-ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7e1746bcc..57d708714 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -374,17 +374,17 @@ stages: - regressions_found=0 - python3 scripts/basop_check_for_changes_in_testcases.py $CSV_BRANCH $CSV_MAIN || regressions_found=$? - - if [ $regressions_found != 0 ]; then + - exit_code=0 + - if [ $regressions_found != 0 ] && ; then - echo "Detected regression wrt to main!" + - exit_code=$EXIT_CODE_NON_BE; - if [ $allow_regressions_flag == 0 ]; then - - exit $EXIT_CODE_FAIL; - - else - - exit $EXIT_CODE_NON_BE; + - exit_code=$EXIT_CODE_FAIL; - fi - fi - - if [ $zero_errors_branch != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_NON_BE; fi - - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi - - exit 0 + - if [ $zero_errors_branch != 1 ]; then echo "Run errors encountered!"; exit_code=$EXIT_CODE_FAIL; fi + - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit_code=$EXIT_CODE_NON_BE; fi + - exit $exit_code after_script: - rm -rf tests/dut tests/ref allow_failure: -- GitLab From b0d5558d5210ef04de5b2196526ed1501c806d57 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 11 Sep 2024 07:53:38 +0200 Subject: [PATCH 02/14] Fix for exit codes, correct one job name --- .gitlab-ci.yml | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 57d708714..b3b54d6dc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -284,17 +284,9 @@ stages: - touch $MERGED_CSV_ARTIFACT_NAME - fi - - if [ $zero_errors != 1 ]; then - - echo "Run errors encountered!" - # TODO: temporary only to not fail pipelines on crashes in encoder - - if [[ "$CI_JOB_NAME" =~ "ivas-pytest-compare_to_ref-short-enc" ]]; then - - exit $EXIT_CODE_NON_BE - - else - - exit $EXIT_CODE_FAIL - - fi - - fi - - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi - - exit 0 + - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit_code=$EXIT_CODE_NON_BE; fi + - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit_code=$EXIT_CODE_FAIL; fi + - exit $exit_code allow_failure: exit_codes: @@ -374,7 +366,8 @@ stages: - regressions_found=0 - python3 scripts/basop_check_for_changes_in_testcases.py $CSV_BRANCH $CSV_MAIN || regressions_found=$? - - exit_code=0 + - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit_code=$EXIT_CODE_NON_BE; fi + - if [ $zero_errors_branch != 1 ]; then echo "Run errors encountered!"; exit_code=$EXIT_CODE_NON_BE; fi - if [ $regressions_found != 0 ] && ; then - echo "Detected regression wrt to main!" - exit_code=$EXIT_CODE_NON_BE; @@ -382,8 +375,6 @@ stages: - exit_code=$EXIT_CODE_FAIL; - fi - fi - - if [ $zero_errors_branch != 1 ]; then echo "Run errors encountered!"; exit_code=$EXIT_CODE_FAIL; fi - - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit_code=$EXIT_CODE_NON_BE; fi - exit $exit_code after_script: - rm -rf tests/dut tests/ref @@ -694,7 +685,7 @@ ivas-pytest-compare_to_ref-short-dec: - LEVEL_SCALING=1.0 <<: *ivas-pytest-anchor -ivas-pytest-compare_to_ref-short-lev-10: +ivas-pytest-compare_to_ref-short-dec-lev-10: extends: - .rules-pytest-to-ref-short - .test-job-linux -- GitLab From ab2e7f1bfdf744189348d647f9f6ef31110658e0 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 11 Sep 2024 08:08:43 +0200 Subject: [PATCH 03/14] Fix in printout to report status of [allow regression] tag --- .gitlab-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b3b54d6dc..e4e9b5d51 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -369,10 +369,12 @@ stages: - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit_code=$EXIT_CODE_NON_BE; fi - if [ $zero_errors_branch != 1 ]; then echo "Run errors encountered!"; exit_code=$EXIT_CODE_NON_BE; fi - if [ $regressions_found != 0 ] && ; then - - echo "Detected regression wrt to main!" - - exit_code=$EXIT_CODE_NON_BE; - if [ $allow_regressions_flag == 0 ]; then + - echo "Detected regression wrt to main, [allow regression] not set!" - exit_code=$EXIT_CODE_FAIL; + - else + - echo "Detected regression wrt to main, [allow regression] set." + - exit_code=$EXIT_CODE_NON_BE; - fi - fi - exit $exit_code -- GitLab From 19169c7d24511f8619d20859ad72eb40c53d4839 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 11 Sep 2024 08:26:19 +0200 Subject: [PATCH 04/14] Fix syntax error in if-statement --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e4e9b5d51..38174ba0b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -368,7 +368,7 @@ stages: - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit_code=$EXIT_CODE_NON_BE; fi - if [ $zero_errors_branch != 1 ]; then echo "Run errors encountered!"; exit_code=$EXIT_CODE_NON_BE; fi - - if [ $regressions_found != 0 ] && ; then + - if [ $regressions_found != 0 ]; then - if [ $allow_regressions_flag == 0 ]; then - echo "Detected regression wrt to main, [allow regression] not set!" - exit_code=$EXIT_CODE_FAIL; -- GitLab From 19e71a1e9164d70d43d6526d712f892fbb336a95 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 11 Sep 2024 08:43:23 +0200 Subject: [PATCH 05/14] Revert change in compare to ref for enc --- .gitlab-ci.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 38174ba0b..a2e936654 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -284,9 +284,17 @@ stages: - touch $MERGED_CSV_ARTIFACT_NAME - fi - - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit_code=$EXIT_CODE_NON_BE; fi - - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit_code=$EXIT_CODE_FAIL; fi - - exit $exit_code + - if [ $zero_errors != 1 ]; then + - echo "Run errors encountered!" + # TODO: temporary only to not fail pipelines on crashes in encoder + - if [[ "$CI_JOB_NAME" =~ "ivas-pytest-compare_to_ref-short-enc" ]]; then + - exit $EXIT_CODE_NON_BE + - else + - exit $EXIT_CODE_FAIL + - fi + - fi + - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 allow_failure: exit_codes: -- GitLab From 30152e46ea8b5107b2954e80e9b840498c3d7e66 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 11 Sep 2024 09:11:54 +0200 Subject: [PATCH 06/14] Fix to allow fail in ivas-pytest-anchor for MR pipelines only --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a2e936654..e74704ed7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -286,8 +286,8 @@ stages: - if [ $zero_errors != 1 ]; then - echo "Run errors encountered!" - # TODO: temporary only to not fail pipelines on crashes in encoder - - if [[ "$CI_JOB_NAME" =~ "ivas-pytest-compare_to_ref-short-enc" ]]; then + # TODO: temporary only to not fail MR pipelines on crashes in encoder + - if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then - exit $EXIT_CODE_NON_BE - else - exit $EXIT_CODE_FAIL -- GitLab From e043418909aaaff15345eeb3f857ff3aadb04d5c Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Wed, 11 Sep 2024 13:54:02 +0200 Subject: [PATCH 07/14] Fix TODO comment. Add pass for identical reports in comparison to main --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e74704ed7..085d7cec6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -286,7 +286,7 @@ stages: - if [ $zero_errors != 1 ]; then - echo "Run errors encountered!" - # TODO: temporary only to not fail MR pipelines on crashes in encoder + # TODO: temporary only to not fail MR pipelines on crashes - if [ $CI_PIPELINE_SOURCE == 'merge_request_event' ]; then - exit $EXIT_CODE_NON_BE - else @@ -385,6 +385,8 @@ stages: - exit_code=$EXIT_CODE_NON_BE; - fi - fi + # If reports are equal, the job will pass + - if [ "$(diff -q $CSV_BRANCH $CSV_MAIN)" == "" ]; then exit_code=0; fi - exit $exit_code after_script: - rm -rf tests/dut tests/ref -- GitLab From c13e7f1f49a9d86df1cf1177f8fc80863db3c2a2 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Thu, 12 Sep 2024 16:47:13 +0200 Subject: [PATCH 08/14] Add check of decoded output signals in comparison to main jobs --- .gitlab-ci.yml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 085d7cec6..11cce4046 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -353,10 +353,13 @@ stages: # this per default builds the branch and the reference and creates the reference outputs - *build-and-create-reference-outputs - exit_code=0 - - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || exit_code=$? + - python3 -m pytest --tb=no $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || exit_code=$? - zero_errors_branch=$(cat $XML_REPORT_BRANCH | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py $XML_REPORT_BRANCH $CSV_BRANCH + # Store branch outputs for comparison + - mv tests/dut tests/dut_branch + # create the summary based on the branch - mkdir $IMAGES_ARTIFACT_NAME - for MEASURE in MLD DIFF SSNR;do python3 scripts/create_histogram_summary.py $CSV_BRANCH $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".csv $IMAGES_ARTIFACT_NAME/summary_"$MEASURE".png --measure $MEASURE; done @@ -367,7 +370,7 @@ stages: - git pull - make clean - make -j - - python3 -m pytest --tb=no $TEST_SUITE -v --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true + - python3 -m pytest --tb=no $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true - python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN ### compare the two csv files for regressions @@ -385,8 +388,16 @@ stages: - exit_code=$EXIT_CODE_NON_BE; - fi - fi - # If reports are equal, the job will pass - - if [ "$(diff -q $CSV_BRANCH $CSV_MAIN)" == "" ]; then exit_code=0; fi + + # If outputs of main and branch are equal and have equal reports, the job will pass. + - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools tests/dut/sba_bs/raw tests/dut_branch/sba_bs/raw + - diff_sba=$? + - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools tests/dut/param_file/dec tests/dut_branch/param_file/dec + - diff_param=$? + - diff $CSV_BRANCH $CSV_MAIN + - diff_report=$? + - if (($diff_param + $diff_sba + $diff_report == 0)); then exit_code=0; fi + - exit $exit_code after_script: - rm -rf tests/dut tests/ref -- GitLab From 8848f282cc61a1e68a6b36029eaac6adea07c0bd Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 13 Sep 2024 08:40:04 +0200 Subject: [PATCH 09/14] in batch_comp_audio output, only show differing files in case of identical outputs, this limits the size of the printout --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 11cce4046..302462992 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -390,9 +390,9 @@ stages: - fi # If outputs of main and branch are equal and have equal reports, the job will pass. - - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools tests/dut/sba_bs/raw tests/dut_branch/sba_bs/raw + - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/sba_bs/raw tests/dut_branch/sba_bs/raw - diff_sba=$? - - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools tests/dut/param_file/dec tests/dut_branch/param_file/dec + - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/param_file/dec tests/dut_branch/param_file/dec - diff_param=$? - diff $CSV_BRANCH $CSV_MAIN - diff_report=$? -- GitLab From febfe9361af1e55b18d684b54eb566c9d58045ef Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Fri, 13 Sep 2024 17:02:26 +0200 Subject: [PATCH 10/14] Do BE comparison first in comparison to main, exit if BE --- .gitlab-ci.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 302462992..f70215e82 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -373,6 +373,15 @@ stages: - python3 -m pytest --tb=no $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true - python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN + # If outputs of main and branch are equal and have equal reports, the job will pass. + - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/sba_bs/raw tests/dut_branch/sba_bs/raw + - diff_sba=$? + - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/param_file/dec tests/dut_branch/param_file/dec + - diff_param=$? + - diff $CSV_BRANCH $CSV_MAIN + - diff_report=$? + - if (($diff_param + $diff_sba + $diff_report == 0)); then exit 0; fi + ### compare the two csv files for regressions - regressions_found=0 - python3 scripts/basop_check_for_changes_in_testcases.py $CSV_BRANCH $CSV_MAIN || regressions_found=$? @@ -389,15 +398,6 @@ stages: - fi - fi - # If outputs of main and branch are equal and have equal reports, the job will pass. - - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/sba_bs/raw tests/dut_branch/sba_bs/raw - - diff_sba=$? - - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/param_file/dec tests/dut_branch/param_file/dec - - diff_param=$? - - diff $CSV_BRANCH $CSV_MAIN - - diff_report=$? - - if (($diff_param + $diff_sba + $diff_report == 0)); then exit_code=0; fi - - exit $exit_code after_script: - rm -rf tests/dut tests/ref -- GitLab From 1d270b4c8d9ba913d9f01d25c840f0a28b184296 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Mon, 16 Sep 2024 07:14:11 +0200 Subject: [PATCH 11/14] Exit with warning for run errors in comparison to main --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f70215e82..5b47debf8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -373,14 +373,14 @@ stages: - python3 -m pytest --tb=no $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true - python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN - # If outputs of main and branch are equal and have equal reports, the job will pass. + # If outputs of main and branch are equal, have equal reports and no run errors were encountered, the job will pass. - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/sba_bs/raw tests/dut_branch/sba_bs/raw - diff_sba=$? - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/param_file/dec tests/dut_branch/param_file/dec - diff_param=$? - diff $CSV_BRANCH $CSV_MAIN - diff_report=$? - - if (($diff_param + $diff_sba + $diff_report == 0)); then exit 0; fi + - if [ $diff_param -eq 0 ] && if [ $diff_sba -eq 0 ] && if [ $diff_report -eq 0 ] && if [ $zero_errors_branch -eq 1 ]; then exit 0; fi ### compare the two csv files for regressions - regressions_found=0 -- GitLab From 1f6cfcb11b68a0330f579673632e552781987ab4 Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Mon, 16 Sep 2024 07:47:57 +0200 Subject: [PATCH 12/14] Add || true to prevent file comparison to fail job --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5b47debf8..c979cc087 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -374,9 +374,9 @@ stages: - python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN # If outputs of main and branch are equal, have equal reports and no run errors were encountered, the job will pass. - - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/sba_bs/raw tests/dut_branch/sba_bs/raw + - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/sba_bs/raw tests/dut_branch/sba_bs/raw || true - diff_sba=$? - - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/param_file/dec tests/dut_branch/param_file/dec + - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/param_file/dec tests/dut_branch/param_file/dec || true - diff_param=$? - diff $CSV_BRANCH $CSV_MAIN - diff_report=$? -- GitLab From 506fb177cd59f9de4c196182fc98ddd851a84caa Mon Sep 17 00:00:00 2001 From: norvell Date: Mon, 16 Sep 2024 06:49:19 +0000 Subject: [PATCH 13/14] Fix syntax error in if statement --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ad2e4bb76..1dc599d7a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -380,7 +380,7 @@ stages: - diff_param=$? - diff $CSV_BRANCH $CSV_MAIN - diff_report=$? - - if [ $diff_param -eq 0 ] && if [ $diff_sba -eq 0 ] && if [ $diff_report -eq 0 ] && if [ $zero_errors_branch -eq 1 ]; then exit 0; fi + - if [ $diff_param -eq 0 ] && [ $diff_sba -eq 0 ] && [ $diff_report -eq 0 ] && [ $zero_errors_branch -eq 1 ]; then exit 0; fi ### compare the two csv files for regressions - regressions_found=0 -- GitLab From 7e9e37ab4fb21c4dd73a1c0c209951bd0954176d Mon Sep 17 00:00:00 2001 From: Erik Norvell Date: Tue, 17 Sep 2024 13:54:41 +0200 Subject: [PATCH 14/14] Fix capturing of output status in file comparison --- .gitlab-ci.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1dc599d7a..c78aa3b27 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -374,13 +374,16 @@ stages: - python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN # If outputs of main and branch are equal, have equal reports and no run errors were encountered, the job will pass. - - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/sba_bs/raw tests/dut_branch/sba_bs/raw || true - - diff_sba=$? - - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/param_file/dec tests/dut_branch/param_file/dec || true - - diff_param=$? - - diff $CSV_BRANCH $CSV_MAIN - - diff_report=$? - - if [ $diff_param -eq 0 ] && [ $diff_sba -eq 0 ] && [ $diff_report -eq 0 ] && [ $zero_errors_branch -eq 1 ]; then exit 0; fi + - diff_sba=0 + - diff_param=0 + - diff_report=0 + - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/sba_bs/raw tests/dut_branch/sba_bs/raw || diff_sba=$? + - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/param_file/dec tests/dut_branch/param_file/dec || diff_param=$? + - diff $CSV_BRANCH $CSV_MAIN || diff_report=$? + - if [ $diff_param -eq 0 ] && [ $diff_sba -eq 0 ] && [ $diff_report -eq 0 ] && [ $zero_errors_branch -eq 1 ]; then + - echo "Output BE to main, identical report and no run errors encountered." + - exit 0; + - fi ### compare the two csv files for regressions - regressions_found=0 -- GitLab