From 3ea6e92d666de55982e2fc5dded4a33000838471 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Feb 2025 13:26:19 +0100 Subject: [PATCH 01/51] add new stage for checking BE with target branch --- .gitlab-ci.yml | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bc48090cd..a19358b6a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -110,6 +110,7 @@ stages: - .pre - prevalidate - build + - check-be - test - deploy @@ -410,6 +411,62 @@ stages: junit: - report-junit.xml +.check-be-to-target-anchor: &check-be-to-target-anchor + stage: check-be + needs: ["build-codec-linux-make"] + timeout: "300 minutes" + variables: + XML_REPORT: "report--$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.xml" + HTML_REPORT: "report--$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.html" + FAILED_TESTCASES_LIST: "failed-testcases.txt" + script: + - set -euxo pipefail + - *print-common-info + - *update-scripts-repo + + - if [ $USE_LTV -eq 1 ]; then + - *update-ltv-repo + - *copy-ltv-files-to-testv-dir + - testcase_timeout=$TESTCASE_TIMEOUT_LTV + - else + - testcase_timeout=$TESTCASE_TIMEOUT_STV + - fi + + - python3 ci/remove_unsupported_testcases.py $PRM_FILES + - if [ $LEVEL_SCALING != "1.0" ];then + - *apply-testv-scaling + - fi + + - *build-and-create-reference-outputs + + - exit_code=0 + - python3 -m pytest --tb=no $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$? + - zero_errors_branch=$(cat $XML_REPORT_BRANCH | grep -c 'errors="0"') || true + + - touch $FAILED_TESTCASES_LIST + - if [ $exit_code -ne 0 ]; then + - exit_code=$EXIT_CODE_NON_BE + - grep "FAILED" pytest_log.txt | sed 's/^FAILED //' | awk -F'::' '{print "\"" $1 "::" $2 "\"" }' > $FAILED_TESTCASES_LIST + - fi + + - exit $exit_code + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA--results" + expire_in: 1 week + when: always + paths: + - $XML_REPORT + - $HTML_REPORT + expose_as: "pytest compare results" + reports: + junit: + - $XML_REPORT_BRANCH + - $XML_REPORT_MAIN + .ivas-pytest-on-merge-request-anchor: &ivas-pytest-on-merge-request-anchor stage: test needs: ["build-codec-linux-make"] @@ -789,6 +846,19 @@ build-codec-windows-msbuild: # Short test jobs that run in merge request pipelines # --------------------------------------------------------------- +### jobs that check for bitexactness of fx encoder and decoder +check-be-to-target-short-enc-0db: + extends: + - .rules-pytest-to-main-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_DECODER_PATH=./IVAS_dec_ref + - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" + - LEVEL_SCALING=1.0 + - SKIP_REGRESSION_CHECK="true" + <<: *check-be-to-target-anchor + ### jobs that test fx encoder -> flt decoder ivas-pytest-compare_to_main-short-enc: extends: -- GitLab From 4823811f127ba86dd516b6f840ca6570ff070cdf Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Feb 2025 13:27:05 +0100 Subject: [PATCH 02/51] disable jobs to save time --- .gitlab-ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a19358b6a..41e70e8eb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -860,7 +860,7 @@ check-be-to-target-short-enc-0db: <<: *check-be-to-target-anchor ### jobs that test fx encoder -> flt decoder -ivas-pytest-compare_to_main-short-enc: +.ivas-pytest-compare_to_main-short-enc: extends: - .rules-pytest-to-main-short - .test-job-linux @@ -872,7 +872,7 @@ ivas-pytest-compare_to_main-short-enc: - SKIP_REGRESSION_CHECK="true" <<: *ivas-pytest-on-merge-request-anchor -ivas-pytest-compare_to_main-short-enc-lev-10: +.ivas-pytest-compare_to_main-short-enc-lev-10: extends: - .rules-pytest-to-main-short - .test-job-linux @@ -884,7 +884,7 @@ ivas-pytest-compare_to_main-short-enc-lev-10: - SKIP_REGRESSION_CHECK="true" <<: *ivas-pytest-on-merge-request-anchor -ivas-pytest-compare_to_main-short-enc-lev+10: +.ivas-pytest-compare_to_main-short-enc-lev+10: extends: - .rules-pytest-to-main-short - .test-job-linux @@ -897,7 +897,7 @@ ivas-pytest-compare_to_main-short-enc-lev+10: <<: *ivas-pytest-on-merge-request-anchor ### jobs that test flt encoder -> fx decoder -ivas-pytest-compare_to_main-short-dec: +.ivas-pytest-compare_to_main-short-dec: extends: - .rules-pytest-to-main-short - .test-job-linux @@ -909,7 +909,7 @@ ivas-pytest-compare_to_main-short-dec: - rm -rf tests/dut tests/ref <<: *ivas-pytest-on-merge-request-anchor -ivas-pytest-compare_to_main-short-dec-lev-10: +.ivas-pytest-compare_to_main-short-dec-lev-10: extends: - .rules-pytest-to-main-short - .test-job-linux @@ -921,7 +921,7 @@ ivas-pytest-compare_to_main-short-dec-lev-10: - rm -rf tests/dut tests/ref <<: *ivas-pytest-on-merge-request-anchor -ivas-pytest-compare_to_main-short-dec-lev+10: +.ivas-pytest-compare_to_main-short-dec-lev+10: extends: - .rules-pytest-to-main-short - .test-job-linux -- GitLab From 74319da31f87d17fdc7ccf301485ffa83e4dfa90 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Feb 2025 13:52:02 +0100 Subject: [PATCH 03/51] set REFERENCE_BRANCH correctly --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 41e70e8eb..d34fbf169 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -416,6 +416,7 @@ stages: needs: ["build-codec-linux-make"] timeout: "300 minutes" variables: + REFERENCE_BRANCH: "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" XML_REPORT: "report--$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.xml" HTML_REPORT: "report--$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.html" FAILED_TESTCASES_LIST: "failed-testcases.txt" @@ -441,7 +442,7 @@ stages: - exit_code=0 - python3 -m pytest --tb=no $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$? - - zero_errors_branch=$(cat $XML_REPORT_BRANCH | grep -c 'errors="0"') || true + - zero_errors_branch=$(cat $XML_REPORT | grep -c 'errors="0"') || true - touch $FAILED_TESTCASES_LIST - if [ $exit_code -ne 0 ]; then -- GitLab From 0c8ec9b91662d084a339802d4b68fb2a19d2bcb6 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Feb 2025 14:28:42 +0100 Subject: [PATCH 04/51] first shorter test version with needs relation --- .gitlab-ci.yml | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d34fbf169..9a8e4cf44 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -185,7 +185,7 @@ stages: - enc_dmx_arg="--compare_enc_dmx" - fi - - python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$? + - python3 -m pytest -k "stereo and at" $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$? .update-scripts-repo: &update-scripts-repo - cd $SCRIPTS_DIR @@ -441,7 +441,7 @@ stages: - *build-and-create-reference-outputs - exit_code=0 - - python3 -m pytest --tb=no $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$? + - python3 -m pytest -k "stereo and at" --tb=no $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$? - zero_errors_branch=$(cat $XML_REPORT | grep -c 'errors="0"') || true - touch $FAILED_TESTCASES_LIST @@ -467,6 +467,7 @@ stages: junit: - $XML_REPORT_BRANCH - $XML_REPORT_MAIN + - $FAILED_TESTCASES_LIST .ivas-pytest-on-merge-request-anchor: &ivas-pytest-on-merge-request-anchor stage: test @@ -485,6 +486,9 @@ stages: - set -euxo pipefail - *print-common-info - *update-scripts-repo + + - ls + - if [ $USE_LTV -eq 1 ]; then - *update-ltv-repo - *copy-ltv-files-to-testv-dir @@ -860,6 +864,22 @@ check-be-to-target-short-enc-0db: - SKIP_REGRESSION_CHECK="true" <<: *check-be-to-target-anchor +check-regressions-short-enc-0db: + stage: test + needs: + - job: "check-be-to-target-short-enc-0db" + artifacts: true + extends: + - .rules-pytest-to-main-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_DECODER_PATH=./IVAS_dec_ref + - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" + - LEVEL_SCALING=1.0 + - SKIP_REGRESSION_CHECK="true" + <<: *ivas-pytest-on-merge-request-anchor + ### jobs that test fx encoder -> flt decoder .ivas-pytest-compare_to_main-short-enc: extends: -- GitLab From cde22561d513cb94e03a99e2d1a478265a4002a2 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Feb 2025 14:35:06 +0100 Subject: [PATCH 05/51] remove needs: in anchor to not overwrite value from job --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9a8e4cf44..05f531b7f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -471,7 +471,6 @@ stages: .ivas-pytest-on-merge-request-anchor: &ivas-pytest-on-merge-request-anchor stage: test - needs: ["build-codec-linux-make"] timeout: "300 minutes" variables: XML_REPORT_BRANCH: "report-junit-branch-$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.xml" -- GitLab From 2a5b563122d4e82278c17d17f22e5b645faf0bf7 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Feb 2025 14:41:07 +0100 Subject: [PATCH 06/51] fix artifacts --- .gitlab-ci.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 05f531b7f..4fa731231 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -450,6 +450,8 @@ stages: - grep "FAILED" pytest_log.txt | sed 's/^FAILED //' | awk -F'::' '{print "\"" $1 "::" $2 "\"" }' > $FAILED_TESTCASES_LIST - fi + - cat $FAILED_TESTCASES_LIST + - exit $exit_code allow_failure: @@ -462,12 +464,12 @@ stages: paths: - $XML_REPORT - $HTML_REPORT + - $FAILED_TESTCASES_LIST expose_as: "pytest compare results" reports: junit: - - $XML_REPORT_BRANCH - - $XML_REPORT_MAIN - - $FAILED_TESTCASES_LIST + - $XML_REPORT + - $XML_REPORT .ivas-pytest-on-merge-request-anchor: &ivas-pytest-on-merge-request-anchor stage: test -- GitLab From 403c61ad3b7ff7f0ff81075c0325592894b6ae32 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Feb 2025 14:50:52 +0100 Subject: [PATCH 07/51] improve failed testcase parsing and add dbg artifact --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4fa731231..747f03d13 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -441,13 +441,13 @@ stages: - *build-and-create-reference-outputs - exit_code=0 - - python3 -m pytest -k "stereo and at" --tb=no $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$? + - python3 -m pytest -k "stereo and at" --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$? - zero_errors_branch=$(cat $XML_REPORT | grep -c 'errors="0"') || true - touch $FAILED_TESTCASES_LIST - if [ $exit_code -ne 0 ]; then - exit_code=$EXIT_CODE_NON_BE - - grep "FAILED" pytest_log.txt | sed 's/^FAILED //' | awk -F'::' '{print "\"" $1 "::" $2 "\"" }' > $FAILED_TESTCASES_LIST + - grep "^FAILED" pytest_log.txt | sed 's/^FAILED //' | awk -F'::' '{print "\"" $1 "::" $2 "\"" }' > $FAILED_TESTCASES_LIST - fi - cat $FAILED_TESTCASES_LIST @@ -465,6 +465,7 @@ stages: - $XML_REPORT - $HTML_REPORT - $FAILED_TESTCASES_LIST + - pytest_log.txt expose_as: "pytest compare results" reports: junit: -- GitLab From 3cae2338706c558cdb1a3b92258076b3c435d959 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Feb 2025 15:02:25 +0100 Subject: [PATCH 08/51] amend pipeline for getting the failed testcases --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 747f03d13..77bba1c95 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -447,7 +447,7 @@ stages: - touch $FAILED_TESTCASES_LIST - if [ $exit_code -ne 0 ]; then - exit_code=$EXIT_CODE_NON_BE - - grep "^FAILED" pytest_log.txt | sed 's/^FAILED //' | awk -F'::' '{print "\"" $1 "::" $2 "\"" }' > $FAILED_TESTCASES_LIST + - grep "^FAILED" pytest_log.txt | sed 's/^FAILED //' | sed 's/] - .*/]/' | awk -F'::' '{print "\"" $1 "::" $2 "\"" }' > $FAILED_TESTCASES_LIST - fi - cat $FAILED_TESTCASES_LIST -- GitLab From 4535432e323fffaf8b8462f4bed0d685b5d187cc Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Feb 2025 15:10:42 +0100 Subject: [PATCH 09/51] check artifacts for BE in comparison job --- .gitlab-ci.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 77bba1c95..b15c5939f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -34,6 +34,7 @@ variables: ENCODER_TEST: "" COMPARE_DMX: "" SKIP_REGRESSION_CHECK: "" + FAILED_TESTCASES_LIST: "failed-testcases.txt" MANUAL_PIPELINE_TYPE: description: "Type for the manual pipeline run. Use 'pytest-compare' to run comparison test against reference float codec." value: 'default' @@ -419,7 +420,6 @@ stages: REFERENCE_BRANCH: "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" XML_REPORT: "report--$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.xml" HTML_REPORT: "report--$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.html" - FAILED_TESTCASES_LIST: "failed-testcases.txt" script: - set -euxo pipefail - *print-common-info @@ -487,9 +487,16 @@ stages: script: - set -euxo pipefail - *print-common-info - - *update-scripts-repo - - ls + - cat $FAILED_TESTCASES_LIST + - if [ -f "$FAILED_TESTCASES_LIST" ]; then + - $TEST_SUITE=$(cat $FAILED_TESTCASES_LIST) + - else + - echo "All tested cases were bit-exact between $CI_MERGE_REQUEST_TARGET_BRANCH_NAME and $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME. No need to check for regressions." + - exit 0 + - fi + + - *update-scripts-repo - if [ $USE_LTV -eq 1 ]; then - *update-ltv-repo -- GitLab From dde36765f44603386812b899a932524d5a30dc09 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Feb 2025 15:22:43 +0100 Subject: [PATCH 10/51] check for file being empty --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6fc3c1fa0..7db27dd63 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -489,7 +489,7 @@ stages: - *print-common-info - cat $FAILED_TESTCASES_LIST - - if [ -f "$FAILED_TESTCASES_LIST" ]; then + - if [ -f "$FAILED_TESTCASES_LIST" ] && [ ! -z "$( ls -A $FAILED_TESTCASES_LIST )" ]; then - $TEST_SUITE=$(cat $FAILED_TESTCASES_LIST) - else - echo "All tested cases were bit-exact between $CI_MERGE_REQUEST_TARGET_BRANCH_NAME and $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME. No need to check for regressions." -- GitLab From 27456c3c28a80e478a94dfc22a4a1d16b1485719 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Feb 2025 15:37:45 +0100 Subject: [PATCH 11/51] fix check for empty file --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7db27dd63..77fb89dbe 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -488,8 +488,7 @@ stages: - set -euxo pipefail - *print-common-info - - cat $FAILED_TESTCASES_LIST - - if [ -f "$FAILED_TESTCASES_LIST" ] && [ ! -z "$( ls -A $FAILED_TESTCASES_LIST )" ]; then + - if [ -s $FAILED_TESTCASES_LIST ]; then - $TEST_SUITE=$(cat $FAILED_TESTCASES_LIST) - else - echo "All tested cases were bit-exact between $CI_MERGE_REQUEST_TARGET_BRANCH_NAME and $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME. No need to check for regressions." -- GitLab From b79cca58f691da72f9ca7cb32548af5ebbded60c Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Feb 2025 15:55:10 +0100 Subject: [PATCH 12/51] [revert-me] force TCX20 to provoke diff for testing --- lib_enc/transient_detection_fx.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib_enc/transient_detection_fx.c b/lib_enc/transient_detection_fx.c index d9859a1a1..b0d131258 100644 --- a/lib_enc/transient_detection_fx.c +++ b/lib_enc/transient_detection_fx.c @@ -804,6 +804,8 @@ void SetTCXModeInfo_ivas_fx( hTcxEnc->tcxMode = TCX_20; move16(); } + + hTcxEnc->tcxMode = TCX_20; } ELSE { -- GitLab From 42a1d25e5b743dfaa4cbd1873b5070caea330930 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Feb 2025 16:05:51 +0100 Subject: [PATCH 13/51] fix variable assignment --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 77fb89dbe..a3c12f6c9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -489,7 +489,7 @@ stages: - *print-common-info - if [ -s $FAILED_TESTCASES_LIST ]; then - - $TEST_SUITE=$(cat $FAILED_TESTCASES_LIST) + - TEST_SUITE=$(cat $FAILED_TESTCASES_LIST) - else - echo "All tested cases were bit-exact between $CI_MERGE_REQUEST_TARGET_BRANCH_NAME and $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME. No need to check for regressions." - exit 0 -- GitLab From a494d1d67767cf35567cef8c7b4203db68df04df Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Feb 2025 16:16:35 +0100 Subject: [PATCH 14/51] remove newlines from artifact file --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a3c12f6c9..dbdd25f75 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -447,7 +447,7 @@ stages: - touch $FAILED_TESTCASES_LIST - if [ $exit_code -ne 0 ]; then - exit_code=$EXIT_CODE_NON_BE - - grep "^FAILED" pytest_log.txt | sed 's/^FAILED //' | sed 's/] - .*/]/' | awk -F'::' '{print "\"" $1 "::" $2 "\"" }' > $FAILED_TESTCASES_LIST + - grep "^FAILED" pytest_log.txt | sed 's/^FAILED //' | sed 's/] - .*/]/' | awk -F'::' '{print "\"" $1 "::" $2 "\"" }' | tr "\n" " " > $FAILED_TESTCASES_LIST - fi - cat $FAILED_TESTCASES_LIST -- GitLab From 3d459969828c4ea73d0220258a190002a2cc7fbe Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Feb 2025 16:24:54 +0100 Subject: [PATCH 15/51] try without additional "" --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dbdd25f75..fcad772f7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -447,7 +447,7 @@ stages: - touch $FAILED_TESTCASES_LIST - if [ $exit_code -ne 0 ]; then - exit_code=$EXIT_CODE_NON_BE - - grep "^FAILED" pytest_log.txt | sed 's/^FAILED //' | sed 's/] - .*/]/' | awk -F'::' '{print "\"" $1 "::" $2 "\"" }' | tr "\n" " " > $FAILED_TESTCASES_LIST + - grep "^FAILED" pytest_log.txt | sed 's/^FAILED //' | sed 's/] - .*/]/' | tr "\n" " " > $FAILED_TESTCASES_LIST - fi - cat $FAILED_TESTCASES_LIST -- GitLab From e0c6fd18548e729d9f00fce04bf099b6e64449ab Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Feb 2025 16:59:25 +0100 Subject: [PATCH 16/51] try with moving the pytest cache instead of file parsing --- .gitlab-ci.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fcad772f7..2c570229b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,6 +35,7 @@ variables: COMPARE_DMX: "" SKIP_REGRESSION_CHECK: "" FAILED_TESTCASES_LIST: "failed-testcases.txt" + PYTEST_CACHE_ARTIFACT: "pytest_cache.zip" MANUAL_PIPELINE_TYPE: description: "Type for the manual pipeline run. Use 'pytest-compare' to run comparison test against reference float codec." value: 'default' @@ -441,6 +442,7 @@ stages: - *build-and-create-reference-outputs - exit_code=0 + - rm -rf .pytest_cache || true - python3 -m pytest -k "stereo and at" --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$? - zero_errors_branch=$(cat $XML_REPORT | grep -c 'errors="0"') || true @@ -452,6 +454,7 @@ stages: - cat $FAILED_TESTCASES_LIST + - zip -r $PYTEST_CACHE_ARTIFACT .pytest_cache - exit $exit_code allow_failure: @@ -466,6 +469,7 @@ stages: - $HTML_REPORT - $FAILED_TESTCASES_LIST - pytest_log.txt + - $PYTEST_CACHE_ARTIFACT expose_as: "pytest compare results" reports: junit: @@ -489,7 +493,9 @@ stages: - *print-common-info - if [ -s $FAILED_TESTCASES_LIST ]; then - - TEST_SUITE=$(cat $FAILED_TESTCASES_LIST) + - rm -rf .pytest_cache || true + - unzip $PYTEST_CACHE_ARTIFACT + - PYTEST_ADDOPTS="--last-failed" - else - echo "All tested cases were bit-exact between $CI_MERGE_REQUEST_TARGET_BRANCH_NAME and $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME. No need to check for regressions." - exit 0 -- GitLab From 6be56cb42c5094400bdba05e8704d138f0531e5a Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Feb 2025 17:22:40 +0100 Subject: [PATCH 17/51] set PYTEST_ADDOPTS as env var --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2c570229b..7ae61dc30 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -495,7 +495,7 @@ stages: - if [ -s $FAILED_TESTCASES_LIST ]; then - rm -rf .pytest_cache || true - unzip $PYTEST_CACHE_ARTIFACT - - PYTEST_ADDOPTS="--last-failed" + - export PYTEST_ADDOPTS=--last-failed - else - echo "All tested cases were bit-exact between $CI_MERGE_REQUEST_TARGET_BRANCH_NAME and $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME. No need to check for regressions." - exit 0 -- GitLab From b48d823bdd916292c018e338c0fe9df8d28340af Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Feb 2025 17:31:42 +0100 Subject: [PATCH 18/51] overwrite cache b4 every pytest call --- .gitlab-ci.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7ae61dc30..5017a4112 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -476,6 +476,12 @@ stages: - $XML_REPORT - $XML_REPORT +.overwrite-pytest-cache-with-artifact: &overwrite-pytest-cache-with-artifact + - if [ -f $PYTEST_CACHE_ARTIFACT ] then + - rm -rf .pytest_cache || true + - unzip $PYTEST_CACHE_ARTIFACT + - fi + .ivas-pytest-on-merge-request-anchor: &ivas-pytest-on-merge-request-anchor stage: test timeout: "300 minutes" @@ -493,8 +499,7 @@ stages: - *print-common-info - if [ -s $FAILED_TESTCASES_LIST ]; then - - rm -rf .pytest_cache || true - - unzip $PYTEST_CACHE_ARTIFACT + - *overwrite-pytest-cache-with-artifact - export PYTEST_ADDOPTS=--last-failed - else - echo "All tested cases were bit-exact between $CI_MERGE_REQUEST_TARGET_BRANCH_NAME and $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME. No need to check for regressions." @@ -524,6 +529,8 @@ stages: # this per default builds the branch and the reference and creates the reference outputs - *build-and-create-reference-outputs - exit_code=0 + # need to restore cache again + - *overwrite-pytest-cache-with-artifact - python3 -m pytest --tb=no $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || exit_code=$? - zero_errors_branch=$(cat $XML_REPORT_BRANCH | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py $XML_REPORT_BRANCH $CSV_BRANCH @@ -541,6 +548,8 @@ stages: - git pull - make clean - make -j + # need to restore cache again + - *overwrite-pytest-cache-with-artifact - python3 -m pytest --tb=no $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true - python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN -- GitLab From b76abea1c07f129a2b4bd72e8907cc432aae5105 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Feb 2025 17:45:48 +0100 Subject: [PATCH 19/51] fix syntax error --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5017a4112..3e034bebd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -477,7 +477,7 @@ stages: - $XML_REPORT .overwrite-pytest-cache-with-artifact: &overwrite-pytest-cache-with-artifact - - if [ -f $PYTEST_CACHE_ARTIFACT ] then + - if [ -f $PYTEST_CACHE_ARTIFACT ]; then - rm -rf .pytest_cache || true - unzip $PYTEST_CACHE_ARTIFACT - fi -- GitLab From 94fe3242b661346909ff5ebb142191d8a6d4fc41 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Feb 2025 17:47:16 +0100 Subject: [PATCH 20/51] [revert-me] deactivate more jobs for faster testing --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3e034bebd..4e5748231 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -817,7 +817,7 @@ build-codec-linux-make: - make -j # ensure that codec builds on linux with instrumentation active -build-codec-linux-instrumented-make: +.build-codec-linux-instrumented-make: rules: - if: $CI_PIPELINE_SOURCE == 'web' - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH @@ -834,7 +834,7 @@ build-codec-linux-instrumented-make: - bash scripts/prepare_instrumentation.sh -m MEM_ONLY -p BASOP - make -j -C $INSTR_DIR -build-codec-linux-debugging-make: +.build-codec-linux-debugging-make: rules: - if: $CI_PIPELINE_SOURCE == 'web' - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH @@ -852,7 +852,7 @@ build-codec-linux-debugging-make: - *activate-debug-mode-info-if-set - make -j -build-codec-windows-msbuild: +.build-codec-windows-msbuild: rules: - if: $CI_PIPELINE_SOURCE == 'web' - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH @@ -1352,7 +1352,7 @@ coverage-test-on-main-scheduled: # --------------------------------------------------------------- # check bitexactness to EVS -be-2-evs-26444: +.be-2-evs-26444: extends: - .test-job-linux rules: -- GitLab From bd5f9b08172ba992243c71f005cc6e58fcdbac45 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Feb 2025 18:00:29 +0100 Subject: [PATCH 21/51] add missing jobs --- .gitlab-ci.yml | 111 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 92 insertions(+), 19 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4e5748231..881b41b8b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -885,13 +885,10 @@ check-be-to-target-short-enc-0db: - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - LEVEL_SCALING=1.0 - SKIP_REGRESSION_CHECK="true" + - rm -rf tests/dut tests/ref <<: *check-be-to-target-anchor -check-regressions-short-enc-0db: - stage: test - needs: - - job: "check-be-to-target-short-enc-0db" - artifacts: true +check-be-to-target-short-enc-+10db: extends: - .rules-pytest-to-main-short - .test-job-linux @@ -899,12 +896,66 @@ check-regressions-short-enc-0db: - USE_LTV=0 - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - - LEVEL_SCALING=1.0 + - LEVEL_SCALING=3.162 - SKIP_REGRESSION_CHECK="true" - <<: *ivas-pytest-on-merge-request-anchor + - rm -rf tests/dut tests/ref + <<: *check-be-to-target-anchor -### jobs that test fx encoder -> flt decoder -.ivas-pytest-compare_to_main-short-enc: +check-be-to-target-short-enc--10db: + extends: + - .rules-pytest-to-main-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_DECODER_PATH=./IVAS_dec_ref + - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" + - LEVEL_SCALING=0.3162 + - SKIP_REGRESSION_CHECK="true" + - rm -rf tests/dut tests/ref + <<: *check-be-to-target-anchor + +check-be-to-target-short-dec-0db: + extends: + - .rules-pytest-to-main-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_ENCODER_PATH=./IVAS_cod_ref + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=1.0 + - rm -rf tests/dut tests/ref + <<: *check-be-to-target-anchor + +check-be-to-target-short-dec-+10db: + extends: + - .rules-pytest-to-main-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_ENCODER_PATH=./IVAS_cod_ref + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=3.162 + - rm -rf tests/dut tests/ref + <<: *check-be-to-target-anchor + +check-be-to-target-short-dec--10db: + extends: + - .rules-pytest-to-main-short + - .test-job-linux + before_script: + - USE_LTV=0 + - DUT_ENCODER_PATH=./IVAS_cod_ref + - TEST_SUITE="$SHORT_TEST_SUITE" + - LEVEL_SCALING=0.3162 + - rm -rf tests/dut tests/ref + <<: *check-be-to-target-anchor + +### jobs that check for regressions on non-BE testcases +check-regressions-short-enc-0db: + stage: test + needs: + - job: "check-be-to-target-short-enc-0db" + artifacts: true extends: - .rules-pytest-to-main-short - .test-job-linux @@ -914,9 +965,14 @@ check-regressions-short-enc-0db: - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - LEVEL_SCALING=1.0 - SKIP_REGRESSION_CHECK="true" + - rm -rf tests/dut tests/ref <<: *ivas-pytest-on-merge-request-anchor -.ivas-pytest-compare_to_main-short-enc-lev-10: +check-regressions-short-enc-+10db: + stage: test + needs: + - job: "check-be-to-target-short-enc-+10db" + artifacts: true extends: - .rules-pytest-to-main-short - .test-job-linux @@ -924,11 +980,16 @@ check-regressions-short-enc-0db: - USE_LTV=0 - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - - LEVEL_SCALING=0.3162 + - LEVEL_SCALING=3.162 - SKIP_REGRESSION_CHECK="true" + - rm -rf tests/dut tests/ref <<: *ivas-pytest-on-merge-request-anchor -.ivas-pytest-compare_to_main-short-enc-lev+10: +check-regressions-short-enc--10db: + stage: test + needs: + - job: "check-be-to-target-short-enc--10db" + artifacts: true extends: - .rules-pytest-to-main-short - .test-job-linux @@ -936,12 +997,16 @@ check-regressions-short-enc-0db: - USE_LTV=0 - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - - LEVEL_SCALING=3.162 + - LEVEL_SCALING=0.3162 - SKIP_REGRESSION_CHECK="true" + - rm -rf tests/dut tests/ref <<: *ivas-pytest-on-merge-request-anchor -### jobs that test flt encoder -> fx decoder -.ivas-pytest-compare_to_main-short-dec: +check-regressions-short-dec-0db: + stage: test + needs: + - job: "check-be-to-target-short-dec-0db" + artifacts: true extends: - .rules-pytest-to-main-short - .test-job-linux @@ -953,7 +1018,11 @@ check-regressions-short-enc-0db: - rm -rf tests/dut tests/ref <<: *ivas-pytest-on-merge-request-anchor -.ivas-pytest-compare_to_main-short-dec-lev-10: +check-regressions-short-enc-+10db: + stage: test + needs: + - job: "check-be-to-target-short-enc-+10db" + artifacts: true extends: - .rules-pytest-to-main-short - .test-job-linux @@ -961,11 +1030,15 @@ check-regressions-short-enc-0db: - USE_LTV=0 - DUT_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$SHORT_TEST_SUITE" - - LEVEL_SCALING=0.3162 + - LEVEL_SCALING=3.162 - rm -rf tests/dut tests/ref <<: *ivas-pytest-on-merge-request-anchor -.ivas-pytest-compare_to_main-short-dec-lev+10: +check-regressions-short-enc--10db: + stage: test + needs: + - job: "check-be-to-target-short-enc--10db" + artifacts: true extends: - .rules-pytest-to-main-short - .test-job-linux @@ -973,7 +1046,7 @@ check-regressions-short-enc-0db: - USE_LTV=0 - DUT_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$SHORT_TEST_SUITE" - - LEVEL_SCALING=3.162 + - LEVEL_SCALING=0.3162 - rm -rf tests/dut tests/ref <<: *ivas-pytest-on-merge-request-anchor -- GitLab From 4fcc08c18f5b08ff9ce4fa602a21d41b58aff97a Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 13 Feb 2025 18:01:49 +0100 Subject: [PATCH 22/51] fix job names --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 881b41b8b..32003043d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1018,10 +1018,10 @@ check-regressions-short-dec-0db: - rm -rf tests/dut tests/ref <<: *ivas-pytest-on-merge-request-anchor -check-regressions-short-enc-+10db: +check-regressions-short-dec-+10db: stage: test needs: - - job: "check-be-to-target-short-enc-+10db" + - job: "check-be-to-target-short-dec-+10db" artifacts: true extends: - .rules-pytest-to-main-short @@ -1034,10 +1034,10 @@ check-regressions-short-enc-+10db: - rm -rf tests/dut tests/ref <<: *ivas-pytest-on-merge-request-anchor -check-regressions-short-enc--10db: +check-regressions-short-dec--10db: stage: test needs: - - job: "check-be-to-target-short-enc--10db" + - job: "check-be-to-target-short-dec--10db" artifacts: true extends: - .rules-pytest-to-main-short -- GitLab From dbb55697955ed456f41e51f63e5498fb9438a941 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Feb 2025 09:52:13 +0100 Subject: [PATCH 23/51] improve reporting in be check job --- .gitlab-ci.yml | 69 +++++++++++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 29 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 32003043d..5007018a3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,6 +35,7 @@ variables: COMPARE_DMX: "" SKIP_REGRESSION_CHECK: "" FAILED_TESTCASES_LIST: "failed-testcases.txt" + ERRORS_TESTCASES_LIST: "errors-testcases.txt" PYTEST_CACHE_ARTIFACT: "pytest_cache.zip" MANUAL_PIPELINE_TYPE: description: "Type for the manual pipeline run. Use 'pytest-compare' to run comparison test against reference float codec." @@ -305,6 +306,14 @@ stages: tags: - ivas-windows +.print-results-banner: &print-results-banner + - set +x + - echo "" + - echo "==================================================================================================================" + - echo "================================================== TEST RESULTS ==================================================" + - echo "==================================================================================================================" + - echo "" + # template for test jobs on linux that need the TESTV_DIR .test-job-linux-needs-testv-dir: extends: .test-job-linux @@ -444,17 +453,37 @@ stages: - exit_code=0 - rm -rf .pytest_cache || true - python3 -m pytest -k "stereo and at" --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$? - - zero_errors_branch=$(cat $XML_REPORT | grep -c 'errors="0"') || true - touch $FAILED_TESTCASES_LIST + - touch $ERRORS_TESTCASES_LIST + - touch $PYTEST_CACHE_ARTIFACT - if [ $exit_code -ne 0 ]; then - exit_code=$EXIT_CODE_NON_BE + - zip -r $PYTEST_CACHE_ARTIFACT .pytest_cache + - grep "^FAILED" pytest_log.txt | sed 's/^FAILED //' | sed 's/] - .*/]/' | tr "\n" " " > $FAILED_TESTCASES_LIST - - fi + - grep "^FAILED" pytest_log.txt | sed 's/^FAILED //' | sed 's/] - .*/]/' > failed_testcases_for_printing.txt + - num_failures=$(wc -l < failed_testcases_for_printing.txt) - - cat $FAILED_TESTCASES_LIST + - grep "^ERROR" pytest_log.txt | sed 's/^ERROR //' | sed 's/] - .*/]/' | tr "\n" " " > $ERRORS_TESTCASES_LIST + - grep "^ERROR" pytest_log.txt | sed 's/^ERROR //' | sed 's/] - .*/]/' > errors_testcases_for_printing.txt + - num_errors=$(wc -l < errors_testcases_for_printing.txt) - - zip -r $PYTEST_CACHE_ARTIFACT .pytest_cache + - *print-results-banner + - echo "Found these $num_failures non-bitexact testcases:" + - cat failed_testcases_for_printing.txt + + - if [ $num_errors -ne 0 ]; then + - exit_code=1 + - echo "There were errors present in the following testcases:" + - cat errors_testcases_for_printing.txt + - fi + + - exit $exit_code + - else + - *print-results-banner + - echo "All testcases are bitexact." + - fi - exit $exit_code allow_failure: @@ -502,7 +531,10 @@ stages: - *overwrite-pytest-cache-with-artifact - export PYTEST_ADDOPTS=--last-failed - else - - echo "All tested cases were bit-exact between $CI_MERGE_REQUEST_TARGET_BRANCH_NAME and $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME. No need to check for regressions." + # turn off echoing back of commands for result printout + - *print-results-banner + - echo "All tested cases were bit-exact between $CI_MERGE_REQUEST_TARGET_BRANCH_NAME and $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME." + - echo "No need to check for regressions. All is fine." - exit 0 - fi @@ -528,10 +560,9 @@ stages: ### run branch first # this per default builds the branch and the reference and creates the reference outputs - *build-and-create-reference-outputs - - exit_code=0 # need to restore cache again - *overwrite-pytest-cache-with-artifact - - python3 -m pytest --tb=no $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || exit_code=$? + - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true - zero_errors_branch=$(cat $XML_REPORT_BRANCH | grep -c 'errors="0"') || true - python3 scripts/parse_xml_report.py $XML_REPORT_BRANCH $CSV_BRANCH @@ -550,34 +581,14 @@ stages: - make -j # need to restore cache again - *overwrite-pytest-cache-with-artifact - - python3 -m pytest --tb=no $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true + - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_MAIN --self-contained-html --junit-xml=$XML_REPORT_MAIN --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true - python3 scripts/parse_xml_report.py $XML_REPORT_MAIN $CSV_MAIN - # If outputs of main and branch are equal, have equal reports and no run errors were encountered, the job will pass. - - diff_sba=0 - - diff_param=0 - - diff_report=0 - # SHORT_TEST_SUITE_ENCODER does not contain test_sba.py. This leads to non-existing output folders being compared and to diff_sba=1. Therefore, this is skipped for the encoder tests - - if [ "$TEST_SUITE" != "$SHORT_TEST_SUITE_ENCODER" ]; then - - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/sba_bs/raw tests/dut_branch/sba_bs/raw || diff_sba=$? - - fi - - python3 scripts/batch_comp_audio.py --tool pyaudio3dtools -sd tests/dut/param_file/dec tests/dut_branch/param_file/dec || diff_param=$? - - diff $CSV_BRANCH $CSV_MAIN || diff_report=$? - - if [ $diff_param -eq 0 ] && [ $diff_sba -eq 0 ] && [ $diff_report -eq 0 ] && [ $zero_errors_branch -eq 1 ]; then - - echo "Output BE to main, identical report and no run errors encountered." - # Add dummy files to avoid warning on missing artifacts - - touch changes_crashes.csv - - touch changes_MLD.csv - - touch changes_MAXIMUM_ABS_DIFF.csv - - touch changes_MIN_SSNR.csv - - touch changes_MIN_ODG.csv - - exit 0; - - fi - ### compare the two csv files for regressions - regressions_found=0 - python3 scripts/basop_check_for_changes_in_testcases.py --xml_report $XML_REPORT_BRANCH $CSV_MAIN $CSV_BRANCH || regressions_found=$? + - exit_code=0 - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit_code=$EXIT_CODE_NON_BE; fi - if [ $zero_errors_branch != 1 ]; then echo "Run errors encountered!"; exit_code=$EXIT_CODE_NON_BE; fi - if [ $regressions_found != 0 ] && [ "$SKIP_REGRESSION_CHECK" != "true" ]; then -- GitLab From 6e747abbf37c5e0cbe41d683cb416c10b83d3d6c Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Feb 2025 10:32:00 +0100 Subject: [PATCH 24/51] condense printout --- .gitlab-ci.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5007018a3..b8b21b72b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -309,10 +309,10 @@ stages: .print-results-banner: &print-results-banner - set +x - echo "" - - echo "==================================================================================================================" - - echo "================================================== TEST RESULTS ==================================================" - - echo "==================================================================================================================" - - echo "" + - > + echo -e "==================================================================================================================\n +================================================== TEST RESULTS ==================================================\n +==================================================================================================================\n" # template for test jobs on linux that need the TESTV_DIR .test-job-linux-needs-testv-dir: @@ -533,8 +533,7 @@ stages: - else # turn off echoing back of commands for result printout - *print-results-banner - - echo "All tested cases were bit-exact between $CI_MERGE_REQUEST_TARGET_BRANCH_NAME and $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME." - - echo "No need to check for regressions. All is fine." + - echo -e "All tested cases were bit-exact between $CI_MERGE_REQUEST_TARGET_BRANCH_NAME and $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME.\nNo need to check for regressions. All is fine." - exit 0 - fi -- GitLab From 0cae04a9ef1b5ceac92c43f42f2c51e70be10c0d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Feb 2025 10:33:44 +0100 Subject: [PATCH 25/51] fix artifact-related zip error --- .gitlab-ci.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b8b21b72b..d8eecaabe 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -454,9 +454,6 @@ stages: - rm -rf .pytest_cache || true - python3 -m pytest -k "stereo and at" --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$? - - touch $FAILED_TESTCASES_LIST - - touch $ERRORS_TESTCASES_LIST - - touch $PYTEST_CACHE_ARTIFACT - if [ $exit_code -ne 0 ]; then - exit_code=$EXIT_CODE_NON_BE - zip -r $PYTEST_CACHE_ARTIFACT .pytest_cache @@ -481,6 +478,10 @@ stages: - exit $exit_code - else + # create empty files to not have errors at artifact stage + - touch $FAILED_TESTCASES_LIST + - touch $ERRORS_TESTCASES_LIST + - touch $PYTEST_CACHE_ARTIFACT - *print-results-banner - echo "All testcases are bitexact." - fi @@ -497,6 +498,7 @@ stages: - $XML_REPORT - $HTML_REPORT - $FAILED_TESTCASES_LIST + - $ERRORS_TESTCASES_LIST - pytest_log.txt - $PYTEST_CACHE_ARTIFACT expose_as: "pytest compare results" -- GitLab From b39991b5fdadac3d4ed624d569d955b5ba187906 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Feb 2025 10:35:17 +0100 Subject: [PATCH 26/51] collapse into one line --- .gitlab-ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d8eecaabe..5ee556e60 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -309,10 +309,7 @@ stages: .print-results-banner: &print-results-banner - set +x - echo "" - - > - echo -e "==================================================================================================================\n -================================================== TEST RESULTS ==================================================\n -==================================================================================================================\n" + - echo -e "==================================================================================================================\n================================================== TEST RESULTS ==================================================\n==================================================================================================================\n" # template for test jobs on linux that need the TESTV_DIR .test-job-linux-needs-testv-dir: -- GitLab From db14b5ea2514f5349ed819e2b27b6052a19dfcb3 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Feb 2025 10:42:12 +0100 Subject: [PATCH 27/51] ignore return codes for grep pipes --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5ee556e60..d672a4b98 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -455,12 +455,12 @@ stages: - exit_code=$EXIT_CODE_NON_BE - zip -r $PYTEST_CACHE_ARTIFACT .pytest_cache - - grep "^FAILED" pytest_log.txt | sed 's/^FAILED //' | sed 's/] - .*/]/' | tr "\n" " " > $FAILED_TESTCASES_LIST - - grep "^FAILED" pytest_log.txt | sed 's/^FAILED //' | sed 's/] - .*/]/' > failed_testcases_for_printing.txt + - grep "^FAILED" pytest_log.txt | sed 's/^FAILED //' | sed 's/] - .*/]/' | tr "\n" " " > $FAILED_TESTCASES_LIST || true + - grep "^FAILED" pytest_log.txt | sed 's/^FAILED //' | sed 's/] - .*/]/' > failed_testcases_for_printing.txt || true - num_failures=$(wc -l < failed_testcases_for_printing.txt) - - grep "^ERROR" pytest_log.txt | sed 's/^ERROR //' | sed 's/] - .*/]/' | tr "\n" " " > $ERRORS_TESTCASES_LIST - - grep "^ERROR" pytest_log.txt | sed 's/^ERROR //' | sed 's/] - .*/]/' > errors_testcases_for_printing.txt + - grep "^ERROR" pytest_log.txt | sed 's/^ERROR //' | sed 's/] - .*/]/' | tr "\n" " " > $ERRORS_TESTCASES_LIST || true + - grep "^ERROR" pytest_log.txt | sed 's/^ERROR //' | sed 's/] - .*/]/' > errors_testcases_for_printing.txt || true - num_errors=$(wc -l < errors_testcases_for_printing.txt) - *print-results-banner -- GitLab From 499c80b726336eb6294211a28ce07ed3cc389b51 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Feb 2025 10:57:43 +0100 Subject: [PATCH 28/51] rework if and result reporting in regression branch --- .gitlab-ci.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d672a4b98..c0e04314d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -587,14 +587,16 @@ stages: - python3 scripts/basop_check_for_changes_in_testcases.py --xml_report $XML_REPORT_BRANCH $CSV_MAIN $CSV_BRANCH || regressions_found=$? - exit_code=0 - - if [ $exit_code -eq 1 ]; then echo "Differences encountered"; exit_code=$EXIT_CODE_NON_BE; fi - - if [ $zero_errors_branch != 1 ]; then echo "Run errors encountered!"; exit_code=$EXIT_CODE_NON_BE; fi - - if [ $regressions_found != 0 ] && [ "$SKIP_REGRESSION_CHECK" != "true" ]; then + - *print-results-banner + - if [ $zero_errors_branch != 1 ]; then + - echo "Run errors encountered!" + - exit_code=$EXIT_CODE_FAIL + - elif [ $regressions_found != 0 ] && [ "$SKIP_REGRESSION_CHECK" != "true" ]; then - if [ $allow_regressions_flag == 0 ]; then - - echo "Detected regression wrt to main, [allow regression] not set!" + - echo "Detected regression wrt to $CI_MERGE_REQUEST_TARGET_BRANCH_NAME, [allow regression] not set!" - exit_code=$EXIT_CODE_FAIL; - else - - echo "Detected regression wrt to main, [allow regression] set." + - echo "Detected regression wrt to $CI_MERGE_REQUEST_TARGET_BRANCH_NAME, [allow regression] set." - exit_code=$EXIT_CODE_NON_BE; - fi - fi -- GitLab From 5eb39f0dd6600569a66c417297df647e0b17aa08 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Feb 2025 11:23:00 +0100 Subject: [PATCH 29/51] remove SKIP_REGrESSION_CHECK --- .gitlab-ci.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c0e04314d..36bb0d5d5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -895,7 +895,6 @@ check-be-to-target-short-enc-0db: - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - LEVEL_SCALING=1.0 - - SKIP_REGRESSION_CHECK="true" - rm -rf tests/dut tests/ref <<: *check-be-to-target-anchor @@ -908,7 +907,6 @@ check-be-to-target-short-enc-+10db: - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - LEVEL_SCALING=3.162 - - SKIP_REGRESSION_CHECK="true" - rm -rf tests/dut tests/ref <<: *check-be-to-target-anchor @@ -921,7 +919,6 @@ check-be-to-target-short-enc--10db: - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - LEVEL_SCALING=0.3162 - - SKIP_REGRESSION_CHECK="true" - rm -rf tests/dut tests/ref <<: *check-be-to-target-anchor @@ -975,7 +972,6 @@ check-regressions-short-enc-0db: - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - LEVEL_SCALING=1.0 - - SKIP_REGRESSION_CHECK="true" - rm -rf tests/dut tests/ref <<: *ivas-pytest-on-merge-request-anchor @@ -992,7 +988,6 @@ check-regressions-short-enc-+10db: - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - LEVEL_SCALING=3.162 - - SKIP_REGRESSION_CHECK="true" - rm -rf tests/dut tests/ref <<: *ivas-pytest-on-merge-request-anchor @@ -1009,7 +1004,6 @@ check-regressions-short-enc--10db: - DUT_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - LEVEL_SCALING=0.3162 - - SKIP_REGRESSION_CHECK="true" - rm -rf tests/dut tests/ref <<: *ivas-pytest-on-merge-request-anchor -- GitLab From 8a5c23e97fd44728ac6ede26212596c0a3c9bd14 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Feb 2025 11:34:47 +0100 Subject: [PATCH 30/51] move printout of regression testcases into results section --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 36bb0d5d5..ec7634a6f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -584,7 +584,7 @@ stages: ### compare the two csv files for regressions - regressions_found=0 - - python3 scripts/basop_check_for_changes_in_testcases.py --xml_report $XML_REPORT_BRANCH $CSV_MAIN $CSV_BRANCH || regressions_found=$? + - python3 scripts/basop_check_for_changes_in_testcases.py --xml_report $XML_REPORT_BRANCH $CSV_MAIN $CSV_BRANCH > regression_log.txt || regressions_found=$? - exit_code=0 - *print-results-banner @@ -592,6 +592,7 @@ stages: - echo "Run errors encountered!" - exit_code=$EXIT_CODE_FAIL - elif [ $regressions_found != 0 ] && [ "$SKIP_REGRESSION_CHECK" != "true" ]; then + - cat regression_log.txt - if [ $allow_regressions_flag == 0 ]; then - echo "Detected regression wrt to $CI_MERGE_REQUEST_TARGET_BRANCH_NAME, [allow regression] not set!" - exit_code=$EXIT_CODE_FAIL; -- GitLab From 9db1630e8057b7f6d16f35371c6dd2675b41672c Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Feb 2025 12:14:58 +0100 Subject: [PATCH 31/51] add infos for local reproduction to check-be tests --- .gitlab-ci.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ec7634a6f..e80445293 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,6 +37,8 @@ variables: FAILED_TESTCASES_LIST: "failed-testcases.txt" ERRORS_TESTCASES_LIST: "errors-testcases.txt" PYTEST_CACHE_ARTIFACT: "pytest_cache.zip" + REF_COMMIT_FILE: "ref-branch-git-sha" + CUT_COMMIT_FILE: "CuT-branch-git-sha" MANUAL_PIPELINE_TYPE: description: "Type for the manual pipeline run. Use 'pytest-compare' to run comparison test against reference float codec." value: 'default' @@ -145,6 +147,7 @@ stages: - fi .build-reference-binaries: &build-reference-binaries + - git rev-parse HEAD > $CUT_COMMIT_FILE - current_commit_sha=$(git rev-parse HEAD) ### build reference binaries - git checkout $REFERENCE_BRANCH @@ -158,6 +161,7 @@ stages: - mv ./IVAS_rend ./IVAS_rend_ref ### Return to current branch - git restore . + - git rev-parse HEAD > $REF_COMMIT_FILE - git checkout $current_commit_sha @@ -467,6 +471,10 @@ stages: - echo "Found these $num_failures non-bitexact testcases:" - cat failed_testcases_for_printing.txt + - echo "Reproduce locally with:" + - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" + - echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH" + - if [ $num_errors -ne 0 ]; then - exit_code=1 - echo "There were errors present in the following testcases:" @@ -498,6 +506,8 @@ stages: - $ERRORS_TESTCASES_LIST - pytest_log.txt - $PYTEST_CACHE_ARTIFACT + - $REF_COMMIT_FILE + - $CUT_COMMIT_FILE expose_as: "pytest compare results" reports: junit: -- GitLab From 5f54c650b822bc251d02df4806786079818e694d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Feb 2025 12:17:25 +0100 Subject: [PATCH 32/51] add infos also to regression check jobs --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e80445293..fd0596d4d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -601,6 +601,9 @@ stages: - if [ $zero_errors_branch != 1 ]; then - echo "Run errors encountered!" - exit_code=$EXIT_CODE_FAIL + - echo "Reproduce locally with:" + - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $ERRORS_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" + - echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $ERRORS_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH" - elif [ $regressions_found != 0 ] && [ "$SKIP_REGRESSION_CHECK" != "true" ]; then - cat regression_log.txt - if [ $allow_regressions_flag == 0 ]; then @@ -610,6 +613,9 @@ stages: - echo "Detected regression wrt to $CI_MERGE_REQUEST_TARGET_BRANCH_NAME, [allow regression] set." - exit_code=$EXIT_CODE_NON_BE; - fi + - echo "Reproduce locally with:" + - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" + - echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH" - fi - exit $exit_code -- GitLab From b8f5e73dc5e8c9383ee681d0ed8fce68aa47e20a Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Feb 2025 12:27:28 +0100 Subject: [PATCH 33/51] change formatting in FAILED_TESTCASE_LIST and ERRORS_TESTCASES_LIST to this should make command pasteable --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fd0596d4d..f351c5c27 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -459,12 +459,12 @@ stages: - exit_code=$EXIT_CODE_NON_BE - zip -r $PYTEST_CACHE_ARTIFACT .pytest_cache - - grep "^FAILED" pytest_log.txt | sed 's/^FAILED //' | sed 's/] - .*/]/' | tr "\n" " " > $FAILED_TESTCASES_LIST || true - - grep "^FAILED" pytest_log.txt | sed 's/^FAILED //' | sed 's/] - .*/]/' > failed_testcases_for_printing.txt || true + - grep "^FAILED" pytest_log.txt | sed "s/^FAILED /'/" | sed "s/] - .*/]'/" | tr "\n" " " > $FAILED_TESTCASES_LIST || true + - grep "^FAILED" pytest_log.txt | sed "s/^FAILED //" | sed "s/] - .*/]/" > failed_testcases_for_printing.txt || true - num_failures=$(wc -l < failed_testcases_for_printing.txt) - - grep "^ERROR" pytest_log.txt | sed 's/^ERROR //' | sed 's/] - .*/]/' | tr "\n" " " > $ERRORS_TESTCASES_LIST || true - - grep "^ERROR" pytest_log.txt | sed 's/^ERROR //' | sed 's/] - .*/]/' > errors_testcases_for_printing.txt || true + - grep "^ERROR" pytest_log.txt | sed "s/^ERROR /'/" | sed "s/] - .*/]'/" | tr "\n" " " > $ERRORS_TESTCASES_LIST || true + - grep "^ERROR" pytest_log.txt | sed "s/^ERROR //" | sed "s/] - .*/]/" > errors_testcases_for_printing.txt || true - num_errors=$(wc -l < errors_testcases_for_printing.txt) - *print-results-banner -- GitLab From 556377cbfec6ae4006d2609e4a6c6f0681223f01 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 14 Feb 2025 12:37:05 +0100 Subject: [PATCH 34/51] add notion about where to find the command lines --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f351c5c27..875a4a478 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -474,7 +474,7 @@ stages: - echo "Reproduce locally with:" - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" - echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH" - + - echo "The individual command lines can be found in the html report in the job artifacts." - if [ $num_errors -ne 0 ]; then - exit_code=1 - echo "There were errors present in the following testcases:" @@ -604,6 +604,7 @@ stages: - echo "Reproduce locally with:" - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $ERRORS_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" - echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $ERRORS_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH" + - echo "The individual command lines can be found in the changes*.csv files in the job artifacts." - elif [ $regressions_found != 0 ] && [ "$SKIP_REGRESSION_CHECK" != "true" ]; then - cat regression_log.txt - if [ $allow_regressions_flag == 0 ]; then @@ -616,6 +617,7 @@ stages: - echo "Reproduce locally with:" - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" - echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH" + - echo "The individual command lines can be found in the changes*.csv files in the job artifacts." - fi - exit $exit_code -- GitLab From 4b7574d6caaa4ee87b5cc11918dbce5415d15d3d Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 17 Feb 2025 12:37:00 +0100 Subject: [PATCH 35/51] add pre and post check for being up-to-date with target --- .gitlab-ci.yml | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 875a4a478..97de3b670 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -117,6 +117,7 @@ stages: - build - check-be - test + - postvalidate - deploy # --------------------------------------------------------------- @@ -229,6 +230,13 @@ stages: - (Get-Content -Path "CMakeLists.txt") -replace '# \(add_compile_options\("\/WX"\)\)', '$1' | Set-Content -Path "CMakeLists.txt" - Get-ChildItem -Path "Workspace_msvc" -Filter "*.vcxproj" | ForEach-Object { (Get-Content -Path $_.FullName) -replace 'false', 'true' | Set-Content -Path $_.FullName } +.rules-merge-request: + extends: .rules-basis + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + .rules-pytest-to-ref-short: rules: - if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "pytest-compare" @@ -774,6 +782,35 @@ uninterruptible: # verification jobs # --------------------------------------------------------------- +branch-is-up-to-date-with-target-pre: + extends: + - .rules-merge-request + stage: prevalidate + needs: [] + tags: + - ivas-basop-linux + script: + - *get-commits-behind-count + - | + if [ $commits_behind_count -ne 0 ]; then + echo -e "Your branch is $commits_behind_count commits behind the target branch, run\n\tgit pull origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME\nto update." + exit 1 + fi + +branch-is-up-to-date-with-target-post: + extends: + - .rules-merge-request + stage: postvalidate + tags: + - ivas-basop-linux + script: + - *get-commits-behind-count + - | + if [ $commits_behind_count -ne 0 ]; then + echo -e "Your branch is $commits_behind_count commits behind the target branch, possibly main changed during your pipeline run, run\n\tgit pull origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME\nto update." + exit 1 + fi + clang-format-check: extends: - .test-job-linux -- GitLab From 543229ce49a777b1cfab2ac3c0f198f6c745b6a9 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 17 Feb 2025 12:40:07 +0100 Subject: [PATCH 36/51] add up-to-date check to BE test as well --- .gitlab-ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 97de3b670..597e15e40 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -441,6 +441,14 @@ stages: HTML_REPORT: "report--$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.html" script: - set -euxo pipefail + + - *get-commits-behind-count + - | + if [ $commits_behind_count -ne 0 ]; then + echo -e "Your branch is $commits_behind_count commits behind the target branch, possibly main changed during your pipeline run. Checking bitexactness now can result in meaningless results. Run\n\t git pull origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME\nto update." + exit 1 + fi + - *print-common-info - *update-scripts-repo -- GitLab From 5ab4502a9c76ada0195f5ad59756b0e8cbc5c12b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 17 Feb 2025 12:42:11 +0100 Subject: [PATCH 37/51] add missing anchor --- .gitlab-ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 597e15e40..bcfed96df 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -217,6 +217,11 @@ stages: - cd $LTV_DIR - git pull - cd - + +.get-commits-behind-count: &get-commits-behind-count + - echo $CI_COMMIT_SHA + - echo $CI_MERGE_REQUEST_TARGET_BRANCH_NAME + - commits_behind_count=$(git rev-list --count $CI_COMMIT_SHA..origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME) .copy-ltv-files-to-testv-dir: ©-ltv-files-to-testv-dir - cp "$LTV_DIR"/*.wav scripts/testv/ -- GitLab From e28286c8c2df0cb223ce0620b28942626f23ad48 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 17 Feb 2025 12:44:54 +0100 Subject: [PATCH 38/51] fix rule template --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bcfed96df..b6be74fd5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -236,11 +236,11 @@ stages: - Get-ChildItem -Path "Workspace_msvc" -Filter "*.vcxproj" | ForEach-Object { (Get-Content -Path $_.FullName) -replace 'false', 'true' | Set-Content -Path $_.FullName } .rules-merge-request: - extends: .rules-basis rules: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' - if: $CI_PIPELINE_SOURCE == 'push' when: never + - when: never .rules-pytest-to-ref-short: rules: -- GitLab From 53228a4ae9aa28723f0059092363dd8f5319a738 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 17 Feb 2025 12:51:33 +0100 Subject: [PATCH 39/51] check for up-to-date with main in regression jobs, too also rename anchor as it is only used for the regressions checks now --- .gitlab-ci.yml | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b6be74fd5..0d757075f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -436,6 +436,14 @@ stages: junit: - report-junit.xml +.check-up-to-date-in-comparison-jobs: &check-up-to-date-in-comparison-jobs + - *get-commits-behind-count + - | + if [ $commits_behind_count -ne 0 ]; then + echo -e "Your branch is $commits_behind_count commits behind the target branch, possibly main changed during your pipeline run. Checking bitexactness or testing for regressions now can result in meaningless results. Run\n\t git pull origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME\nto update." + exit 1 + fi + .check-be-to-target-anchor: &check-be-to-target-anchor stage: check-be needs: ["build-codec-linux-make"] @@ -445,16 +453,11 @@ stages: XML_REPORT: "report--$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.xml" HTML_REPORT: "report--$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.html" script: - - set -euxo pipefail + - *check-up-to-date-in-comparison-jobs + - *print-common-info - - *get-commits-behind-count - - | - if [ $commits_behind_count -ne 0 ]; then - echo -e "Your branch is $commits_behind_count commits behind the target branch, possibly main changed during your pipeline run. Checking bitexactness now can result in meaningless results. Run\n\t git pull origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME\nto update." - exit 1 - fi + - set -euxo pipefail - - *print-common-info - *update-scripts-repo - if [ $USE_LTV -eq 1 ]; then @@ -541,7 +544,7 @@ stages: - unzip $PYTEST_CACHE_ARTIFACT - fi -.ivas-pytest-on-merge-request-anchor: &ivas-pytest-on-merge-request-anchor +.check-regressions-pytest-anchor: &check-regressions-pytest-anchor stage: test timeout: "300 minutes" variables: @@ -554,9 +557,11 @@ stages: IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME" SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html" script: - - set -euxo pipefail + - *check-up-to-date-in-comparison-jobs - *print-common-info + - set -euxo pipefail + - if [ -s $FAILED_TESTCASES_LIST ]; then - *overwrite-pytest-cache-with-artifact - export PYTEST_ADDOPTS=--last-failed @@ -1042,7 +1047,7 @@ check-regressions-short-enc-0db: - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - LEVEL_SCALING=1.0 - rm -rf tests/dut tests/ref - <<: *ivas-pytest-on-merge-request-anchor + <<: *check-regressions-pytest-anchor check-regressions-short-enc-+10db: stage: test @@ -1058,7 +1063,7 @@ check-regressions-short-enc-+10db: - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - LEVEL_SCALING=3.162 - rm -rf tests/dut tests/ref - <<: *ivas-pytest-on-merge-request-anchor + <<: *check-regressions-pytest-anchor check-regressions-short-enc--10db: stage: test @@ -1074,7 +1079,7 @@ check-regressions-short-enc--10db: - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - LEVEL_SCALING=0.3162 - rm -rf tests/dut tests/ref - <<: *ivas-pytest-on-merge-request-anchor + <<: *check-regressions-pytest-anchor check-regressions-short-dec-0db: stage: test @@ -1090,7 +1095,7 @@ check-regressions-short-dec-0db: - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=1.0 - rm -rf tests/dut tests/ref - <<: *ivas-pytest-on-merge-request-anchor + <<: *check-regressions-pytest-anchor check-regressions-short-dec-+10db: stage: test @@ -1106,7 +1111,7 @@ check-regressions-short-dec-+10db: - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=3.162 - rm -rf tests/dut tests/ref - <<: *ivas-pytest-on-merge-request-anchor + <<: *check-regressions-pytest-anchor check-regressions-short-dec--10db: stage: test @@ -1122,7 +1127,7 @@ check-regressions-short-dec--10db: - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=0.3162 - rm -rf tests/dut tests/ref - <<: *ivas-pytest-on-merge-request-anchor + <<: *check-regressions-pytest-anchor # --------------------------------------------------------------- # Short test jobs for running from web interface or schedule -- GitLab From 978268149b7ce3d432ac43b6f34721f6ea731a5c Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 17 Feb 2025 12:53:09 +0100 Subject: [PATCH 40/51] add commit files to regression check artifacts --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0d757075f..5bd1627ae 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -665,6 +665,8 @@ stages: - $CSV_MAIN - $SUMMARY_HTML_ARTIFACT_NAME - $IMAGES_ARTIFACT_NAME + - $REF_COMMIT_FILE + - $CUT_COMMIT_FILE - changes_crashes.csv - changes_MLD.csv - changes_MAXIMUM_ABS_DIFF.csv -- GitLab From 752cb148959cac020d7a6bdca4580e54f18f8842 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 17 Feb 2025 12:57:33 +0100 Subject: [PATCH 41/51] remove empty needs in build stage --- .gitlab-ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5bd1627ae..aeb3c601b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -312,13 +312,11 @@ stages: .build-job-linux: stage: build timeout: "2 minutes" - needs: [] tags: - ivas-basop-linux .build-job-windows: stage: build - needs: [] timeout: "4 minutes" tags: - ivas-windows -- GitLab From 61fb4a0eaa696effa2b3cb2ce8b59d4b5c0b9cd8 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 17 Feb 2025 14:06:47 +0100 Subject: [PATCH 42/51] Revert "[revert-me] deactivate more jobs for faster testing" This reverts commit 94fe3242b661346909ff5ebb142191d8a6d4fc41. --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aeb3c601b..6961113f4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -902,7 +902,7 @@ build-codec-linux-make: - make -j # ensure that codec builds on linux with instrumentation active -.build-codec-linux-instrumented-make: +build-codec-linux-instrumented-make: rules: - if: $CI_PIPELINE_SOURCE == 'web' - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH @@ -919,7 +919,7 @@ build-codec-linux-make: - bash scripts/prepare_instrumentation.sh -m MEM_ONLY -p BASOP - make -j -C $INSTR_DIR -.build-codec-linux-debugging-make: +build-codec-linux-debugging-make: rules: - if: $CI_PIPELINE_SOURCE == 'web' - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH @@ -937,7 +937,7 @@ build-codec-linux-make: - *activate-debug-mode-info-if-set - make -j -.build-codec-windows-msbuild: +build-codec-windows-msbuild: rules: - if: $CI_PIPELINE_SOURCE == 'web' - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH @@ -1504,7 +1504,7 @@ coverage-test-on-main-scheduled: # --------------------------------------------------------------- # check bitexactness to EVS -.be-2-evs-26444: +be-2-evs-26444: extends: - .test-job-linux rules: -- GitLab From 09823bfe6741dfcec344878c7120320f4e3c594a Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 17 Feb 2025 14:07:31 +0100 Subject: [PATCH 43/51] Revert "[revert-me] force TCX20 to provoke diff for testing" This reverts commit b79cca58f691da72f9ca7cb32548af5ebbded60c. --- lib_enc/transient_detection_fx.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib_enc/transient_detection_fx.c b/lib_enc/transient_detection_fx.c index b0d131258..d9859a1a1 100644 --- a/lib_enc/transient_detection_fx.c +++ b/lib_enc/transient_detection_fx.c @@ -804,8 +804,6 @@ void SetTCXModeInfo_ivas_fx( hTcxEnc->tcxMode = TCX_20; move16(); } - - hTcxEnc->tcxMode = TCX_20; } ELSE { -- GitLab From 3af8351a7f5d0794817df190871841d59233e272 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 17 Feb 2025 14:12:11 +0100 Subject: [PATCH 44/51] remove pytest filter which was used for testing only --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6961113f4..d0b4e4d6a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -193,7 +193,7 @@ stages: - enc_dmx_arg="--compare_enc_dmx" - fi - - python3 -m pytest -k "stereo and at" $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$? + - python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH || exit_code=$? .update-scripts-repo: &update-scripts-repo - cd $SCRIPTS_DIR @@ -475,7 +475,7 @@ stages: - exit_code=0 - rm -rf .pytest_cache || true - - python3 -m pytest -k "stereo and at" --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$? + - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$? - if [ $exit_code -ne 0 ]; then - exit_code=$EXIT_CODE_NON_BE -- GitLab From 960fffa18703d87baedeb23b2a12f9c21fd04860 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 17 Feb 2025 15:05:12 +0100 Subject: [PATCH 45/51] check for up to date branch after build of reference --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d0b4e4d6a..8e1d38a6f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -438,6 +438,7 @@ stages: - *get-commits-behind-count - | if [ $commits_behind_count -ne 0 ]; then + set +x echo -e "Your branch is $commits_behind_count commits behind the target branch, possibly main changed during your pipeline run. Checking bitexactness or testing for regressions now can result in meaningless results. Run\n\t git pull origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME\nto update." exit 1 fi @@ -451,7 +452,6 @@ stages: XML_REPORT: "report--$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.xml" HTML_REPORT: "report--$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.html" script: - - *check-up-to-date-in-comparison-jobs - *print-common-info - set -euxo pipefail @@ -472,6 +472,7 @@ stages: - fi - *build-and-create-reference-outputs + - *check-up-to-date-in-comparison-jobs - exit_code=0 - rm -rf .pytest_cache || true @@ -555,7 +556,6 @@ stages: IMAGES_ARTIFACT_NAME: "images_$CI_JOB_NAME" SUMMARY_HTML_ARTIFACT_NAME: "summary_$CI_JOB_NAME.html" script: - - *check-up-to-date-in-comparison-jobs - *print-common-info - set -euxo pipefail @@ -592,6 +592,7 @@ stages: ### run branch first # this per default builds the branch and the reference and creates the reference outputs - *build-and-create-reference-outputs + - *check-up-to-date-in-comparison-jobs # need to restore cache again - *overwrite-pytest-cache-with-artifact - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT_BRANCH --self-contained-html --junit-xml=$XML_REPORT_BRANCH --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout || true -- GitLab From 2de341b9b6f0b35595bcfe85b8746366f7202e6e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 17 Feb 2025 15:12:35 +0100 Subject: [PATCH 46/51] add --show_improvements arg to basop_check_for_changes.py --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8e1d38a6f..f0dfa5668 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -619,7 +619,7 @@ stages: ### compare the two csv files for regressions - regressions_found=0 - - python3 scripts/basop_check_for_changes_in_testcases.py --xml_report $XML_REPORT_BRANCH $CSV_MAIN $CSV_BRANCH > regression_log.txt || regressions_found=$? + - python3 scripts/basop_check_for_changes_in_testcases.py --show_improvements --xml_report $XML_REPORT_BRANCH $CSV_MAIN $CSV_BRANCH > regression_log.txt || regressions_found=$? - exit_code=0 - *print-results-banner -- GitLab From 10c03e0a51a27a6e37d269f7ecb930926a407a6e Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Thu, 20 Feb 2025 14:58:30 +0100 Subject: [PATCH 47/51] fix order of arguments in basop_check_for_changes.py script --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f3f527373..51d5bc42e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -619,7 +619,7 @@ stages: ### compare the two csv files for regressions - regressions_found=0 - - python3 scripts/basop_check_for_changes_in_testcases.py --show_improvements --xml_report $XML_REPORT_BRANCH $CSV_MAIN $CSV_BRANCH > regression_log.txt || regressions_found=$? + - python3 scripts/basop_check_for_changes_in_testcases.py --show_improvements --xml_report $XML_REPORT_BRANCH $CSV_BRANCH $CSV_MAIN > regression_log.txt || regressions_found=$? - exit_code=0 - *print-results-banner -- GitLab From ff8a621a25b32357e22ed3f256ef62cfea74f66b Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 21 Feb 2025 14:27:21 +0100 Subject: [PATCH 48/51] fix check-be tests comparison was always done with full chain BASOP codec, not using the flt ref en/decoder where applicable --- .gitlab-ci.yml | 81 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 56 insertions(+), 25 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 51d5bc42e..65ae34075 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ variables: TESTV_DIR: "/usr/local/testv" LTV_DIR: "/usr/local/ltv" EVS_BE_TEST_DIR_BASOP: "/usr/local/be_2_evs_basop" - REFERENCE_BRANCH: "ivas-float-update" + FLOAT_REF_BRANCH: "ivas-float-update" BUILD_OUTPUT: "build_output.txt" SCRIPTS_DIR: "/usr/local/scripts" EXIT_CODE_NON_BE: 123 @@ -17,6 +17,8 @@ variables: DUT_DECODER_PATH: "./IVAS_dec" REF_ENCODER_PATH: "./IVAS_cod_ref" REF_DECODER_PATH: "./IVAS_dec_ref" + MERGE_TARGET_ENCODER_PATH: "./IVAS_cod_merge_target" + MERGE_TARGET_DECODER_PATH: "./IVAS_dec_merge_target" LEVEL_SCALING: "1.0" IVAS_PIPELINE_NAME: '' BASOP_CI_BRANCH_PC_REPO: "basop-ci-branch" @@ -37,8 +39,9 @@ variables: FAILED_TESTCASES_LIST: "failed-testcases.txt" ERRORS_TESTCASES_LIST: "errors-testcases.txt" PYTEST_CACHE_ARTIFACT: "pytest_cache.zip" - REF_COMMIT_FILE: "ref-branch-git-sha" - CUT_COMMIT_FILE: "CuT-branch-git-sha" + FLOAT_REF_COMMIT_FILE: "float-ref-git-sha" + CUT_COMMIT_FILE: "CuT-git-sha" + MERGE_TARGET_COMMIT_FILE: "merge-target-git-sha" MANUAL_PIPELINE_TYPE: description: "Type for the manual pipeline run. Use 'pytest-compare' to run comparison test against reference float codec." value: 'default' @@ -147,14 +150,13 @@ stages: - sed -i.bak -e "s/\/\*\ *\(#define\ *DEBUG_MODE_INFO\ *\)\*\//\1/g" lib_com/options.h - fi -.build-reference-binaries: &build-reference-binaries +.build-float-ref-binaries: &build-float-ref-binaries - git rev-parse HEAD > $CUT_COMMIT_FILE - current_commit_sha=$(git rev-parse HEAD) ### build reference binaries - - git checkout $REFERENCE_BRANCH + - git checkout $FLOAT_REF_BRANCH - git pull - *activate-debug-mode-info-if-set - - cat lib_com/options.h - make clean - make -j - mv ./IVAS_cod ./$REF_ENCODER_PATH @@ -162,20 +164,35 @@ stages: - mv ./IVAS_rend ./IVAS_rend_ref ### Return to current branch - git restore . - - git rev-parse HEAD > $REF_COMMIT_FILE + - git rev-parse HEAD > $FLOAT_REF_COMMIT_FILE - git checkout $current_commit_sha +.build-merge-target-binaries: &build-merge-target-binaries + - current_commit_sha=$(git rev-parse HEAD) + ### build merge target binaries + - git checkout $CI_MERGE_REQUEST_TARGET_BRANCH_NAME + - git pull + - *activate-debug-mode-info-if-set + - make clean + - make -j + - mv ./IVAS_cod ./$MERGE_TARGET_ENCODER_PATH + - mv ./IVAS_dec ./$MERGE_TARGET_DECODER_PATH + - mv ./IVAS_rend ./IVAS_rend_merge_target + ### Return to current branch + - git restore . + - git rev-parse HEAD > $MERGE_TARGET_COMMIT_FILE + - git checkout $current_commit_sha -.build-reference-and-dut-binaries: &build-reference-and-dut-binaries +.build-float-ref-and-dut-binaries: &build-float-ref-and-dut-binaries ### build reference binaries - - *build-reference-binaries + - *build-float-ref-binaries ### build dut binaries - *activate-debug-mode-info-if-set - make clean - make -j -.build-and-create-reference-outputs: &build-and-create-reference-outputs - - *build-reference-and-dut-binaries +.build-and-create-float-ref-outputs: &build-and-create-float-ref-outputs + - *build-float-ref-and-dut-binaries ### prepare pytest # create short test vectors @@ -365,7 +382,7 @@ stages: - if [ "$COMPARE_DMX" = "true" ] || [ "$ENCODER_TEST" = "true" ]; then - BUILD_WITH_DEBUG_MODE_INFO="true" - fi - - *build-and-create-reference-outputs + - *build-and-create-float-ref-outputs - comp_args="--mld --ssnr --odg" - if [ "$ENCODER_TEST" = "true" ]; then @@ -448,15 +465,16 @@ stages: needs: ["build-codec-linux-make"] timeout: "300 minutes" variables: - REFERENCE_BRANCH: "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" XML_REPORT: "report--$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.xml" HTML_REPORT: "report--$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.html" + PYTEST_LOG_TARGET_BRANCH: "pytest-log-$CI_MERGE_REQUEST_TARGET_BRANCH_NAME.txt" script: - *print-common-info - set -euxo pipefail - *update-scripts-repo + - python3 tests/create_short_testvectors.py - if [ $USE_LTV -eq 1 ]; then - *update-ltv-repo @@ -471,12 +489,16 @@ stages: - *apply-testv-scaling - fi - - *build-and-create-reference-outputs + - *build-float-ref-binaries + - *build-merge-target-binaries - *check-up-to-date-in-comparison-jobs + - exit_code_target=0 + - python3 -m pytest $TEST_SUITE -v --update_ref 1 --create_ref -n auto --ref_encoder_path $MERGE_TARGET_ENCODER_PATH --ref_decoder_path $MERGE_TARGET_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH > $PYTEST_LOG_TARGET_BRANCH || exit_code_target=$? + - exit_code=0 - rm -rf .pytest_cache || true - - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$? + - python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --mld --ssnr --odg --ref_encoder_path $MERGE_TARGET_ENCODER_PATH --ref_decoder_path $MERGE_TARGET_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt || exit_code=$? - if [ $exit_code -ne 0 ]; then - exit_code=$EXIT_CODE_NON_BE @@ -495,7 +517,7 @@ stages: - cat failed_testcases_for_printing.txt - echo "Reproduce locally with:" - - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" + - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $FLOAT_REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" - echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH" - echo "The individual command lines can be found in the html report in the job artifacts." - if [ $num_errors -ne 0 ]; then @@ -529,8 +551,10 @@ stages: - $ERRORS_TESTCASES_LIST - pytest_log.txt - $PYTEST_CACHE_ARTIFACT - - $REF_COMMIT_FILE + - $FLOAT_REF_COMMIT_FILE - $CUT_COMMIT_FILE + - $MERGE_TARGET_COMMIT_FILE + - $PYTEST_LOG_TARGET_BRANCH expose_as: "pytest compare results" reports: junit: @@ -591,7 +615,7 @@ stages: ### run branch first # this per default builds the branch and the reference and creates the reference outputs - - *build-and-create-reference-outputs + - *build-and-create-float-ref-outputs - *check-up-to-date-in-comparison-jobs # need to restore cache again - *overwrite-pytest-cache-with-artifact @@ -627,7 +651,7 @@ stages: - echo "Run errors encountered!" - exit_code=$EXIT_CODE_FAIL - echo "Reproduce locally with:" - - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $ERRORS_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" + - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $FLOAT_REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $ERRORS_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" - echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $ERRORS_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH" - echo "The individual command lines can be found in the changes*.csv files in the job artifacts." - elif [ $regressions_found != 0 ] && [ "$SKIP_REGRESSION_CHECK" != "true" ]; then @@ -640,7 +664,7 @@ stages: - exit_code=$EXIT_CODE_NON_BE; - fi - echo "Reproduce locally with:" - - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" + - echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $FLOAT_REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH" - echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH" - echo "The individual command lines can be found in the changes*.csv files in the job artifacts." - fi @@ -664,8 +688,9 @@ stages: - $CSV_MAIN - $SUMMARY_HTML_ARTIFACT_NAME - $IMAGES_ARTIFACT_NAME - - $REF_COMMIT_FILE + - $FLOAT_REF_COMMIT_FILE - $CUT_COMMIT_FILE + - $MERGE_TARGET_COMMIT_FILE - changes_crashes.csv - changes_MLD.csv - changes_MAXIMUM_ABS_DIFF.csv @@ -695,7 +720,7 @@ stages: - *copy-ltv-files-to-testv-dir - python3 ci/remove_unsupported_testcases.py $PRM_FILES - - *build-reference-binaries + - *build-float-ref-binaries - set -euxo pipefail - make_args="CLANG=$CLANG_NUM" - if [[ $CLANG_NUM == 3 ]]; then @@ -740,7 +765,7 @@ stages: - if [ $LEVEL_SCALING != "1.0" ];then - *apply-testv-scaling - fi - - *build-reference-and-dut-binaries + - *build-float-ref-and-dut-binaries ### run pytest - exit_code=0 @@ -975,6 +1000,7 @@ check-be-to-target-short-enc-0db: before_script: - USE_LTV=0 - DUT_DECODER_PATH=./IVAS_dec_ref + - MERGE_TARGET_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - LEVEL_SCALING=1.0 - rm -rf tests/dut tests/ref @@ -987,6 +1013,7 @@ check-be-to-target-short-enc-+10db: before_script: - USE_LTV=0 - DUT_DECODER_PATH=./IVAS_dec_ref + - MERGE_TARGET_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - LEVEL_SCALING=3.162 - rm -rf tests/dut tests/ref @@ -999,6 +1026,7 @@ check-be-to-target-short-enc--10db: before_script: - USE_LTV=0 - DUT_DECODER_PATH=./IVAS_dec_ref + - MERGE_TARGET_DECODER_PATH=./IVAS_dec_ref - TEST_SUITE="$SHORT_TEST_SUITE_ENCODER" - LEVEL_SCALING=0.3162 - rm -rf tests/dut tests/ref @@ -1011,6 +1039,7 @@ check-be-to-target-short-dec-0db: before_script: - USE_LTV=0 - DUT_ENCODER_PATH=./IVAS_cod_ref + - MERGE_TARGET_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=1.0 - rm -rf tests/dut tests/ref @@ -1023,6 +1052,7 @@ check-be-to-target-short-dec-+10db: before_script: - USE_LTV=0 - DUT_ENCODER_PATH=./IVAS_cod_ref + - MERGE_TARGET_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=3.162 - rm -rf tests/dut tests/ref @@ -1035,6 +1065,7 @@ check-be-to-target-short-dec--10db: before_script: - USE_LTV=0 - DUT_ENCODER_PATH=./IVAS_cod_ref + - MERGE_TARGET_ENCODER_PATH=./IVAS_cod_ref - TEST_SUITE="$SHORT_TEST_SUITE" - LEVEL_SCALING=0.3162 - rm -rf tests/dut tests/ref @@ -1466,7 +1497,7 @@ coverage-test-on-main-scheduled: - *update-scripts-repo - *update-ltv-repo - *copy-ltv-files-to-testv-dir - - *build-reference-binaries + - *build-float-ref-binaries # Build DuT binaries with GCOV - make clean - make GCOV=1 -j @@ -1665,7 +1696,7 @@ voip-be-on-merge-request: - *print-common-info - *update-scripts-repo - *update-ltv-repo - - *build-reference-and-dut-binaries + - *build-float-ref-and-dut-binaries - *complexity-measurements-setup - which coan artifacts: -- GitLab From 548bdacb70227acf2f10c292563f021c385ce1c7 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 21 Feb 2025 14:38:17 +0100 Subject: [PATCH 49/51] add missing build for branch --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 65ae34075..292d8ae41 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -491,6 +491,8 @@ stages: - *build-float-ref-binaries - *build-merge-target-binaries + - make clean + - make -j - *check-up-to-date-in-comparison-jobs - exit_code_target=0 -- GitLab From e463cc92c149c82a8142eb0045a072ed5cca73f3 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Fri, 21 Feb 2025 15:41:28 +0100 Subject: [PATCH 50/51] run prepare_combined_files script --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 292d8ae41..1ea012c2a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -485,6 +485,8 @@ stages: - fi - python3 ci/remove_unsupported_testcases.py $PRM_FILES + - python3 scripts/prepare_combined_format_inputs.py + - if [ $LEVEL_SCALING != "1.0" ];then - *apply-testv-scaling - fi -- GitLab From e21996d8675465373eb719b2aed585878a2aa0c3 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Mon, 24 Feb 2025 11:23:37 +0100 Subject: [PATCH 51/51] fix bug in renaming of the codec binaries for use by pytest --- .gitlab-ci.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1ea012c2a..a7f044dad 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,12 +13,20 @@ variables: SHORT_TEST_SUITE_ENCODER: "tests/codec_be_on_mr_nonselection/test_param_file.py --param_file scripts/config/self_test_basop_encoder.prm" LONG_TEST_SUITE_ENCODER: "tests/codec_be_on_mr_nonselection/test_param_file.py --param_file scripts/config/self_test_ltv_basop_encoder.prm" TEST_SUITE: "" + # These path variables are used by the pytest calls. + # They can be overwritten in the job templates to e.g. only test encoder or decoder in the chain DUT_ENCODER_PATH: "./IVAS_cod" DUT_DECODER_PATH: "./IVAS_dec" REF_ENCODER_PATH: "./IVAS_cod_ref" REF_DECODER_PATH: "./IVAS_dec_ref" MERGE_TARGET_ENCODER_PATH: "./IVAS_cod_merge_target" MERGE_TARGET_DECODER_PATH: "./IVAS_dec_merge_target" + # These path variables are used for building the binaries + # They should never be overwritten! + REF_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_cod_ref" + REF_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_dec_ref" + MERGE_TARGET_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_cod_merge_target" + MERGE_TARGET_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY: "./IVAS_dec_merge_target" LEVEL_SCALING: "1.0" IVAS_PIPELINE_NAME: '' BASOP_CI_BRANCH_PC_REPO: "basop-ci-branch" @@ -159,8 +167,8 @@ stages: - *activate-debug-mode-info-if-set - make clean - make -j - - mv ./IVAS_cod ./$REF_ENCODER_PATH - - mv ./IVAS_dec ./$REF_DECODER_PATH + - mv ./IVAS_cod ./$REF_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY + - mv ./IVAS_dec ./$REF_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY - mv ./IVAS_rend ./IVAS_rend_ref ### Return to current branch - git restore . @@ -175,8 +183,8 @@ stages: - *activate-debug-mode-info-if-set - make clean - make -j - - mv ./IVAS_cod ./$MERGE_TARGET_ENCODER_PATH - - mv ./IVAS_dec ./$MERGE_TARGET_DECODER_PATH + - mv ./IVAS_cod ./$MERGE_TARGET_ENCODER_PATH_FOR_BUILD_DO_NOT_MODIFY + - mv ./IVAS_dec ./$MERGE_TARGET_DECODER_PATH_FOR_BUILD_DO_NOT_MODIFY - mv ./IVAS_rend ./IVAS_rend_merge_target ### Return to current branch - git restore . -- GitLab