diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0b2b3ddff9e5a94a176be61202901490412ab75b..27d3656175e2d393ad5a30a505036223ef20aefa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,6 +44,52 @@ stages: .get-previous-merge-commit-sha: &get-previous-merge-commit-sha - previous_merge_commit=$(git --no-pager log --merges HEAD~1 -n 1 --pretty=format:%H) +.merge_request_comparison_setup: &merge_request_comparison_setup + ### build test binaries, initial clean for paranoia reasons + - make clean + - mkdir build + - cd build + - cmake .. + - make -j + - mv IVAS_cod ../IVAS_cod_test + - mv IVAS_dec ../IVAS_dec_test + - cd .. + - rm -rf build/* + + ### store the current commit hash + - source_branch_commit_sha=$(git rev-parse HEAD) + + ### checkout version to compare against + # first delete local target branch to avoid conflicts when branch is cached and there are merge conflicts during fetching + # depending on chaching, the branch may not be there, so prevent failure of this command -> should maybe be done smarter later + - git branch -D $CI_MERGE_REQUEST_TARGET_BRANCH_NAME || true + # needed when depth is lower than the number of commits in the branch + - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:$CI_MERGE_REQUEST_TARGET_BRANCH_NAME + + ### compare to last target branch commit before pipeline was created + - target_commit=$(git log $CI_MERGE_REQUEST_TARGET_BRANCH_NAME -1 --oneline --before=${CI_PIPELINE_CREATED_AT} --format=%H) + - git checkout $target_commit + + ### build reference binaries + - cd build + - cmake .. + - make -j + - mv IVAS_cod ../IVAS_cod_ref + - mv IVAS_dec ../IVAS_dec_ref + - cd .. + + # rename test binaries back + - mv IVAS_cod_test IVAS_cod + - mv IVAS_dec_test IVAS_dec + + ### re-checkout the commit from the source branch to have up-to-date self_test.py and scripts/testv (and actually everything) + - git checkout $source_branch_commit_sha + +.merge-request-comparison-check: &merge-request-comparison-check + - if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "Non-bitexact cases without non-BE tag encountered!"; exit $EXIT_CODE_FAIL; fi + - if [ $exit_code -eq 1 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; exit $EXIT_CODE_NON_BE; fi + - exit 0 # --------------------------------------------------------------- # Job templates @@ -275,7 +321,7 @@ asan-on-merge-request-linux: # compare bit exactness between target and source branch -pytest-on-merge-request: +ivas-pytest-on-merge-request: extends: - .test-job-linux - .rules-merge-request @@ -284,85 +330,62 @@ pytest-on-merge-request: timeout: "10 minutes" script: - *print-common-info - ### build test binaries, initial clean for paranoia reasons - - make clean - - mkdir build - - cd build - - cmake .. - - make -j - - mv IVAS_cod ../IVAS_cod_test - - mv IVAS_dec ../IVAS_dec_test - - cd .. - - rm -rf build/* - - ### store the current commit hash - - source_branch_commit_sha=$(git rev-parse HEAD) - - ### checkout version to compare against - # first delete local target branch to avoid conflicts when branch is cached and there are merge conflicts during fetching - # depending on chaching, the branch may not be there, so prevent failure of this command -> should maybe be done smarter later - - git branch -D $CI_MERGE_REQUEST_TARGET_BRANCH_NAME || true - # needed when depth is lower than the number of commits in the branch - - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:$CI_MERGE_REQUEST_TARGET_BRANCH_NAME - - ### compare to last target branch commit before pipeline was created - - target_commit=$(git log $CI_MERGE_REQUEST_TARGET_BRANCH_NAME -1 --oneline --before=${CI_PIPELINE_CREATED_AT} --format=%H) - - git checkout $target_commit - - ### build reference binaries - - cd build - - cmake .. - - make -j - - mv IVAS_cod ../IVAS_cod_ref - - mv IVAS_dec ../IVAS_dec_ref - - cd .. - - ### re-checkout the commit from the source branch to have up-to-date test scripts and test vectors (and actually everything) - - git checkout $source_branch_commit_sha + - *merge_request_comparison_setup # some helper variables - "|| true" to prevent failures from grep not finding anything - - evs_non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[evs[ -]*non[ -]*be\]") || true - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[non[ -]*be\]") || true - - expected_nonbe_1=0 - - expected_nonbe_2=0 - - fail_1=0 - - fail_2=0 ### prepare pytest # create short test vectors - python3 tests/create_short_testvectors.py - # rename test binaries back - - mv IVAS_cod_test IVAS_cod - - mv IVAS_dec_test IVAS_dec # create references - python3 -m pytest tests -v --update_ref 1 -m create_ref - python3 -m pytest tests -v --update_ref 1 -m create_ref_part2 - - python3 -m pytest tests/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm ### run pytest - exit_code=0 - python3 -m pytest tests -v --junit-xml=report-junit.xml || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - - if [ $zero_errors != 1 ]; then echo "Run errors in pytest"; fail_1=1; fi + - *merge-request-comparison-check + + allow_failure: + exit_codes: + - 123 + artifacts: + name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" + when: always + paths: + - report-junit.xml + expose_as: 'pytest ivas results' + reports: + junit: + - report-junit.xml + +evs-pytest-on-merge-request: + extends: + - .test-job-linux + - .rules-merge-request + stage: compare + needs: [ "build-codec-linux-cmake", "codec-smoke-test" ] + timeout: "10 minutes" + script: + - *print-common-info + - *merge_request_comparison_setup - - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures without non-BE tag encountered"; fail_1=1; fi - - if [ $exit_code -eq 1 ] && [ $non_be_flag != 0 ]; then echo "pytest run had failures with non-BE tag encountered"; expected_nonbe_1=1; fi + # some helper variables - "|| true" to prevent failures from grep not finding anything + - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[evs[ -]*non[ -]*be\]") || true + + ### prepare pytest + # create references + - python3 -m pytest tests/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm ### run pytest for EVS cases - exit_code=0 - python3 -m pytest tests/test_param_file.py -v --param_file scripts/config/self_test_evs.prm --junit-xml=report-junit-evs.xml || exit_code=$? - zero_errors=$(cat report-junit-evs.xml | grep -c 'errors="0"') || true - - if [ $zero_errors != 1 ]; then echo "Run errors in pytest for EVS"; fail_2=1; fi - - - if [ $exit_code -eq 1 ] && [ $evs_non_be_flag == 0 ]; then echo "Non-bitexact EVS cases without EVS-non-BE tag encountered"; fail_2=1; fi - - if [ $exit_code -eq 1 ] && [ $evs_non_be_flag != 0 ]; then echo "Non-bitexact EVS cases with EVS-non-BE tag encountered"; expected_nonbe_2=1; fi - - # Check results from both tests - - if [ $fail_1 -eq 1 ] || [ $fail_2 -eq 1 ]; then exit $EXIT_CODE_FAIL; fi - - if [ $expected_nonbe_1 -eq 1 ] || [ $expected_nonbe_2 -eq 1 ]; then exit $EXIT_CODE_NON_BE; fi - - exit 0 + - *merge-request-comparison-check allow_failure: exit_codes: @@ -371,12 +394,10 @@ pytest-on-merge-request: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" when: always paths: - - report-junit.xml - report-junit-evs.xml - expose_as: 'pytest results' + expose_as: 'pytest evs results' reports: junit: - - report-junit.xml - report-junit-evs.xml