From 1145b9b285afa59fd0efa80bf48ba4909aaeff05 Mon Sep 17 00:00:00 2001 From: norvell Date: Wed, 28 Sep 2022 16:51:53 +0000 Subject: [PATCH 1/5] Split self-test-on-merge-request into three separate jobs --- .gitlab-ci.yml | 190 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 124 insertions(+), 66 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cbd61ad2dc..bd2f89c554 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,6 +41,43 @@ stages: .get-previous-merge-commit-sha: &get-previous-merge-commit-sha - previous_merge_commit=$(git --no-pager log --merges HEAD~1 -n 1 --pretty=format:%H) +.merge_request_comparison_setup: &merge_request_comparison_setup + ### build test binaries, initial clean for paranoia reasons + - make clean + - mkdir build + - cd build + - cmake .. + - make -j + - mv IVAS_cod ../IVAS_cod_test + - mv IVAS_dec ../IVAS_dec_test + - cd .. + - rm -rf build/* + + ### store the current commit hash + - source_branch_commit_sha=$(git rev-parse HEAD) + + ### checkout version to compare against + # first delete local target branch to avoid conflicts when branch is cached and there are merge conflicts during fetching + # depending on chaching, the branch may not be there, so prevent failure of this command -> should maybe be done smarter later + - git branch -D $CI_MERGE_REQUEST_TARGET_BRANCH_NAME || true + # needed when depth is lower than the number of commits in the branch + - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:$CI_MERGE_REQUEST_TARGET_BRANCH_NAME + + ### compare to last target branch commit before pipeline was created + - target_commit=$(git log $CI_MERGE_REQUEST_TARGET_BRANCH_NAME -1 --oneline --before=${CI_PIPELINE_CREATED_AT} --format=%H) + - git checkout $target_commit + + ### build reference binaries + - cd build + - cmake .. + - make -j + - mv IVAS_cod ../IVAS_cod_ref + - mv IVAS_dec ../IVAS_dec_ref + - cd .. + + ### re-checkout the commit from the source branch to have up-to-date self_test.py and scripts/testv (and actually everything) + - git checkout $source_branch_commit_sha + # --------------------------------------------------------------- # Job templates @@ -252,8 +289,8 @@ asan-on-merge-request-linux: expose_as: 'Asan selftest results' -# compare bit exactness between target and source branch -self-test-on-merge-request: +# compare bit exactness between target and source branch -- IVAS self_test cases +self-test-ivas-on-merge-request: extends: - .test-job-linux - .rules-merge-request @@ -262,89 +299,68 @@ self-test-on-merge-request: timeout: "10 minutes" script: - *print-common-info - ### build test binaries, initial clean for paranoia reasons - - make clean - - mkdir build - - cd build - - cmake .. - - make -j - - mv IVAS_cod ../IVAS_cod_test - - mv IVAS_dec ../IVAS_dec_test - - cd .. - - rm -rf build/* - - ### store the current commit hash - - source_branch_commit_sha=$(git rev-parse HEAD) - - ### checkout version to compare against - # first delete local target branch to avoid conflicts when branch is cached and there are merge conflicts during fetching - # depending on chaching, the branch may not be there, so prevent failure of this command -> should maybe be done smarter later - - git branch -D $CI_MERGE_REQUEST_TARGET_BRANCH_NAME || true - # needed when depth is lower than the number of commits in the branch - - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:$CI_MERGE_REQUEST_TARGET_BRANCH_NAME - - ### compare to last target branch commit before pipeline was created - - target_commit=$(git log $CI_MERGE_REQUEST_TARGET_BRANCH_NAME -1 --oneline --before=${CI_PIPELINE_CREATED_AT} --format=%H) - - git checkout $target_commit - - ### build reference binaries - - cd build - - cmake .. - - make -j - - mv IVAS_cod ../IVAS_cod_ref - - mv IVAS_dec ../IVAS_dec_ref - - cd .. - - ### re-checkout the commit from the source branch to have up-to-date self_test.py and scripts/testv (and actually everything) - - git checkout $source_branch_commit_sha + - *merge_request_comparison_setup ### run selftest - ls -altr scripts/testv - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test | tee test_output.txt - - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test scripts/config/self_test_evs.prm | tee test_output_evs.txt + ### analyse test output # some helper variables - "|| true" to prevent failures from grep not finding anything - - evs_non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[evs[ -]*non[ -]*be\]") || true - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[non[ -]*be\]") || true - run_errors=$(cat test_output.txt test_output_evs.txt | grep -c "test conditions had run errors") || true - bitexact=$(cat test_output.txt | grep -c "All [0-9]* tests are bitexact") || true - - bitexact_evs=$(cat test_output_evs.txt | grep -c "All [0-9]* tests are bitexact") || true - EXIT_CODE_NON_BE=123 - EXIT_CODE_FAIL=1 - - expected_nonbe_1=0 - - expected_nonbe_2=0 - - expected_nonbe_3=0 - - fail_1=0 - - fail_2=0 - - fail_3=0 # check for crashes during the test, if any happened, fail the test - - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; fail_1=1; fi + - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; exit $EXIT_CODE_FAIL; fi # check for non bitexact output and store exit code to also always run the SBA pytest - - if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; fail_1=1; fi - - if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; expected_nonbe_1=1; fi + - if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; exit $EXIT_CODE_FAIL; fi + - if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + when: always + paths: + - test_output.txt + - test_output_evs.txt + - scripts/test/logs/ + - scripts/ref/logs/ + expose_as: 'Self test IVAS results' - # check for non bitexact EVS output - - if [ $bitexact_evs == 0 ] && [ $evs_non_be_flag == 0 ] ; then echo "Non-bitexact EVS cases without EVS-non-BE tag encountered"; fail_2=1; fi - - if [ $bitexact_evs == 0 ] && [ $evs_non_be_flag != 0 ] ; then echo "Non-bitexact EVS cases with EVS-non-BE tag encountered"; expected_nonbe_2=1; fi +# compare bit exactness between target and source branch -- SBA self_test cases +self-test-sba-on-merge-request: + extends: + - .test-job-linux + - .rules-merge-request + stage: compare + needs: [ "build-codec-linux-cmake", "codec-smoke-test" ] + timeout: "10 minutes" + script: + - *print-common-info + - *merge_request_comparison_setup ### run SBA pytest + - EXIT_CODE_NON_BE=123 + - EXIT_CODE_FAIL=1 - exit_code=0 - python3 ./scripts/ivas_pytests/self_test_b.py --encref IVAS_cod_ref --decref IVAS_dec_ref --encdut IVAS_cod_test --decdut IVAS_dec_test || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - if [ $zero_errors != 1 ]; then echo "Run errors in SBA pytest"; fail_3=1; fi - - - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures without non-BE tag encountered"; fail_3=1; fi - - if [ $exit_code -eq 1 ] && [ $non_be_flag == 1 ]; then echo "pytest run had failures with non-BE tag encountered"; expected_nonbe_3=1; fi - - # Check results from all three tests - - if [ $fail_1 -eq 1 ] || [ $fail_2 -eq 1 ] || [ $fail_3 -eq 1 ]; then exit $EXIT_CODE_FAIL; fi - - if [ $expected_nonbe_1 -eq 1 ] || [ $expected_nonbe_2 -eq 1 ] || [ $expected_nonbe_3 -eq 1 ]; then exit $EXIT_CODE_NON_BE; fi + - if [ $zero_errors != 1 ]; then echo "Run errors in SBA pytest"; exit $EXIT_CODE_FAIL; fi + + - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures without non-BE tag encountered"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -eq 1 ] && [ $non_be_flag == 1 ]; then echo "pytest run had failures with non-BE tag encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 - + allow_failure: exit_codes: - 123 @@ -352,16 +368,58 @@ self-test-on-merge-request: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" when: always paths: - - test_output.txt - - test_output_evs.txt - - scripts/test/logs/ - - scripts/ref/logs/ - report-junit.xml - expose_as: 'Self test results' + expose_as: 'SBA test results' reports: junit: report-junit.xml +# compare bit exactness between target and source branch -- EVS self_test cases +self-test-evs-on-merge-request: + extends: + - .test-job-linux + - .rules-merge-request + stage: compare + needs: [ "build-codec-linux-cmake", "codec-smoke-test" ] + timeout: "10 minutes" + script: + - *print-common-info + - *merge_request_comparison_setup + + ### run selftest + - ls -altr scripts/testv + - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test scripts/config/self_test_evs.prm | tee test_output_evs.txt + ### analyse test output + + # some helper variables - "|| true" to prevent failures from grep not finding anything + - evs_non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[evs[ -]*non[ -]*be\]") || true + - run_errors=$(cat test_output_evs.txt | grep -c "test conditions had run errors") || true + - bitexact_evs=$(cat test_output_evs.txt | grep -c "All [0-9]* tests are bitexact") || true + - EXIT_CODE_NON_BE=123 + - EXIT_CODE_FAIL=1 + + # check for crashes during the test, if any happened, fail the test + - if [ $run_errors != 0 ] ; then echo "Run errors in self_test_evs.py"; exit $EXIT_CODE_FAIL; fi + + # check for non bitexact EVS output + - if [ $bitexact_evs == 0 ] && [ $evs_non_be_flag == 0 ] ; then echo "Non-bitexact EVS cases without EVS-non-BE tag encountered"; exit $EXIT_CODE_FAIL; fi + - if [ $bitexact_evs == 0 ] && [ $evs_non_be_flag != 0 ] ; then echo "Non-bitexact EVS cases with EVS-non-BE tag encountered"; exit $EXIT_CODE_NON_BE; fi + + - exit 0 + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + when: always + paths: + - test_output_evs.txt + - scripts/test/logs/ + - scripts/ref/logs/ + expose_as: 'Self test EVS results' + + # --------------------------------------------------------------- # Test jobs for main branch # --------------------------------------------------------------- -- GitLab From b3fbb38a428c0df2a4d5b0cd01a0456800feae50 Mon Sep 17 00:00:00 2001 From: norvell Date: Wed, 28 Sep 2022 17:06:25 +0000 Subject: [PATCH 2/5] Correct some errors in self_test_b.py test and remove self_test_evs.txt artifact from IVAS test job. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bd2f89c554..5eed1cb726 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -330,7 +330,6 @@ self-test-ivas-on-merge-request: when: always paths: - test_output.txt - - test_output_evs.txt - scripts/test/logs/ - scripts/ref/logs/ expose_as: 'Self test IVAS results' @@ -353,6 +352,7 @@ self-test-sba-on-merge-request: - exit_code=0 - python3 ./scripts/ivas_pytests/self_test_b.py --encref IVAS_cod_ref --decref IVAS_dec_ref --encdut IVAS_cod_test --decdut IVAS_dec_test || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[non[ -]*be\]") || true - if [ $zero_errors != 1 ]; then echo "Run errors in SBA pytest"; exit $EXIT_CODE_FAIL; fi -- GitLab From 87614157bd3c6345e8e3132fd072209939d39e00 Mon Sep 17 00:00:00 2001 From: norvell Date: Tue, 18 Oct 2022 09:34:33 +0000 Subject: [PATCH 3/5] Change name of merge request comparison jobs to have ivas- and evs- in the beginning. This way it is more visible in the pipeline overview where the names are shortened. --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d028ce9356..1e35837393 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -312,7 +312,7 @@ asan-on-merge-request-linux: # compare bit exactness between target and source branch -pytest-on-merge-request-ivas: +ivas-pytest-on-merge-request: extends: - .test-job-linux - .rules-merge-request @@ -362,7 +362,7 @@ pytest-on-merge-request-ivas: junit: - report-junit.xml -pytest-on-merge-request-evs: +evs-pytest-on-merge-request: extends: - .test-job-linux - .rules-merge-request -- GitLab From 8ca6b98bf9a8dce1d2a0390a6703a6134ba94699 Mon Sep 17 00:00:00 2001 From: norvell Date: Wed, 19 Oct 2022 15:15:48 +0000 Subject: [PATCH 4/5] Clean out two unused variables --- .gitlab-ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1e35837393..7948146179 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -325,8 +325,6 @@ ivas-pytest-on-merge-request: # some helper variables - "|| true" to prevent failures from grep not finding anything - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[non[ -]*be\]") || true - - expected_nonbe_1=0 - - fail_1=0 ### prepare pytest # create short test vectors -- GitLab From e36827f9f82b4c59f897ec1c31cca5df5f157d58 Mon Sep 17 00:00:00 2001 From: knj Date: Wed, 26 Oct 2022 08:51:29 +0200 Subject: [PATCH 5/5] put more common parts of the pytest jobs into yaml anchors --- .gitlab-ci.yml | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7948146179..27d3656175 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -78,9 +78,18 @@ stages: - mv IVAS_dec ../IVAS_dec_ref - cd .. + # rename test binaries back + - mv IVAS_cod_test IVAS_cod + - mv IVAS_dec_test IVAS_dec + ### re-checkout the commit from the source branch to have up-to-date self_test.py and scripts/testv (and actually everything) - git checkout $source_branch_commit_sha +.merge-request-comparison-check: &merge-request-comparison-check + - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "Non-bitexact cases without non-BE tag encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -eq 1 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 # --------------------------------------------------------------- # Job templates @@ -329,9 +338,6 @@ ivas-pytest-on-merge-request: ### prepare pytest # create short test vectors - python3 tests/create_short_testvectors.py - # rename test binaries back - - mv IVAS_cod_test IVAS_cod - - mv IVAS_dec_test IVAS_dec # create references - python3 -m pytest tests -v --update_ref 1 -m create_ref - python3 -m pytest tests -v --update_ref 1 -m create_ref_part2 @@ -341,11 +347,7 @@ ivas-pytest-on-merge-request: - python3 -m pytest tests -v --junit-xml=report-junit.xml || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - if [ $zero_errors != 1 ]; then echo "Run errors in pytest"; exit $EXIT_CODE_FAIL; fi - - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures without non-BE tag encountered"; exit $EXIT_CODE_FAIL; fi - - if [ $exit_code -eq 1 ] && [ $non_be_flag != 0 ]; then echo "pytest run had failures with non-BE tag encountered"; exit $EXIT_CODE_NON_BE; fi - - - exit 0 + - *merge-request-comparison-check allow_failure: exit_codes: @@ -372,12 +374,9 @@ evs-pytest-on-merge-request: - *merge_request_comparison_setup # some helper variables - "|| true" to prevent failures from grep not finding anything - - evs_non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[evs[ -]*non[ -]*be\]") || true + - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[evs[ -]*non[ -]*be\]") || true ### prepare pytest - # rename test binaries back - - mv IVAS_cod_test IVAS_cod - - mv IVAS_dec_test IVAS_dec # create references - python3 -m pytest tests/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm @@ -386,10 +385,7 @@ evs-pytest-on-merge-request: - python3 -m pytest tests/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --junit-xml=report-junit-evs.xml || exit_code=$? - zero_errors=$(cat report-junit-evs.xml | grep -c 'errors="0"') || true - - if [ $zero_errors != 1 ]; then echo "Run errors in pytest for EVS"; exit $EXIT_CODE_FAIL; fi - - if [ $exit_code -eq 1 ] && [ $evs_non_be_flag == 0 ]; then echo "Non-bitexact EVS cases without EVS-non-BE tag encountered"; exit $EXIT_CODE_FAIL; fi - - if [ $exit_code -eq 1 ] && [ $evs_non_be_flag != 0 ]; then echo "Non-bitexact EVS cases with EVS-non-BE tag encountered"; exit $EXIT_CODE_NON_BE; fi - - exit 0 + - *merge-request-comparison-check allow_failure: exit_codes: -- GitLab