Loading .gitlab-ci.yml +124 −66 Original line number Diff line number Diff line Loading @@ -41,6 +41,43 @@ stages: .get-previous-merge-commit-sha: &get-previous-merge-commit-sha - previous_merge_commit=$(git --no-pager log --merges HEAD~1 -n 1 --pretty=format:%H) .merge_request_comparison_setup: &merge_request_comparison_setup ### build test binaries, initial clean for paranoia reasons - make clean - mkdir build - cd build - cmake .. - make -j - mv IVAS_cod ../IVAS_cod_test - mv IVAS_dec ../IVAS_dec_test - cd .. - rm -rf build/* ### store the current commit hash - source_branch_commit_sha=$(git rev-parse HEAD) ### checkout version to compare against # first delete local target branch to avoid conflicts when branch is cached and there are merge conflicts during fetching # depending on chaching, the branch may not be there, so prevent failure of this command -> should maybe be done smarter later - git branch -D $CI_MERGE_REQUEST_TARGET_BRANCH_NAME || true # needed when depth is lower than the number of commits in the branch - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:$CI_MERGE_REQUEST_TARGET_BRANCH_NAME ### compare to last target branch commit before pipeline was created - target_commit=$(git log $CI_MERGE_REQUEST_TARGET_BRANCH_NAME -1 --oneline --before=${CI_PIPELINE_CREATED_AT} --format=%H) - git checkout $target_commit ### build reference binaries - cd build - cmake .. - make -j - mv IVAS_cod ../IVAS_cod_ref - mv IVAS_dec ../IVAS_dec_ref - cd .. ### re-checkout the commit from the source branch to have up-to-date self_test.py and scripts/testv (and actually everything) - git checkout $source_branch_commit_sha # --------------------------------------------------------------- # Job templates Loading Loading @@ -252,8 +289,8 @@ asan-on-merge-request-linux: expose_as: 'Asan selftest results' # compare bit exactness between target and source branch self-test-on-merge-request: # compare bit exactness between target and source branch -- IVAS self_test cases self-test-ivas-on-merge-request: extends: - .test-job-linux - .rules-merge-request Loading @@ -262,87 +299,66 @@ self-test-on-merge-request: timeout: "10 minutes" script: - *print-common-info ### build test binaries, initial clean for paranoia reasons - make clean - mkdir build - cd build - cmake .. - make -j - mv IVAS_cod ../IVAS_cod_test - mv IVAS_dec ../IVAS_dec_test - cd .. - rm -rf build/* ### store the current commit hash - source_branch_commit_sha=$(git rev-parse HEAD) ### checkout version to compare against # first delete local target branch to avoid conflicts when branch is cached and there are merge conflicts during fetching # depending on chaching, the branch may not be there, so prevent failure of this command -> should maybe be done smarter later - git branch -D $CI_MERGE_REQUEST_TARGET_BRANCH_NAME || true # needed when depth is lower than the number of commits in the branch - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:$CI_MERGE_REQUEST_TARGET_BRANCH_NAME ### compare to last target branch commit before pipeline was created - target_commit=$(git log $CI_MERGE_REQUEST_TARGET_BRANCH_NAME -1 --oneline --before=${CI_PIPELINE_CREATED_AT} --format=%H) - git checkout $target_commit ### build reference binaries - cd build - cmake .. - make -j - mv IVAS_cod ../IVAS_cod_ref - mv IVAS_dec ../IVAS_dec_ref - cd .. ### re-checkout the commit from the source branch to have up-to-date self_test.py and scripts/testv (and actually everything) - git checkout $source_branch_commit_sha - *merge_request_comparison_setup ### run selftest - ls -altr scripts/testv - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test | tee test_output.txt - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test scripts/config/self_test_evs.prm | tee test_output_evs.txt ### analyse test output # some helper variables - "|| true" to prevent failures from grep not finding anything - evs_non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[evs[ -]*non[ -]*be\]") || true - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[non[ -]*be\]") || true - run_errors=$(cat test_output.txt test_output_evs.txt | grep -c "test conditions had run errors") || true - bitexact=$(cat test_output.txt | grep -c "All [0-9]* tests are bitexact") || true - bitexact_evs=$(cat test_output_evs.txt | grep -c "All [0-9]* tests are bitexact") || true - EXIT_CODE_NON_BE=123 - EXIT_CODE_FAIL=1 - expected_nonbe_1=0 - expected_nonbe_2=0 - expected_nonbe_3=0 - fail_1=0 - fail_2=0 - fail_3=0 # check for crashes during the test, if any happened, fail the test - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; fail_1=1; fi - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; exit $EXIT_CODE_FAIL; fi # check for non bitexact output and store exit code to also always run the SBA pytest - if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; fail_1=1; fi - if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; expected_nonbe_1=1; fi - if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; exit $EXIT_CODE_FAIL; fi - if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; exit $EXIT_CODE_NON_BE; fi - exit 0 # check for non bitexact EVS output - if [ $bitexact_evs == 0 ] && [ $evs_non_be_flag == 0 ] ; then echo "Non-bitexact EVS cases without EVS-non-BE tag encountered"; fail_2=1; fi - if [ $bitexact_evs == 0 ] && [ $evs_non_be_flag != 0 ] ; then echo "Non-bitexact EVS cases with EVS-non-BE tag encountered"; expected_nonbe_2=1; fi allow_failure: exit_codes: - 123 artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" when: always paths: - test_output.txt - test_output_evs.txt - scripts/test/logs/ - scripts/ref/logs/ expose_as: 'Self test IVAS results' # compare bit exactness between target and source branch -- SBA self_test cases self-test-sba-on-merge-request: extends: - .test-job-linux - .rules-merge-request stage: compare needs: [ "build-codec-linux-cmake", "codec-smoke-test" ] timeout: "10 minutes" script: - *print-common-info - *merge_request_comparison_setup ### run SBA pytest - EXIT_CODE_NON_BE=123 - EXIT_CODE_FAIL=1 - exit_code=0 - python3 ./scripts/ivas_pytests/self_test_b.py --encref IVAS_cod_ref --decref IVAS_dec_ref --encdut IVAS_cod_test --decdut IVAS_dec_test || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - if [ $zero_errors != 1 ]; then echo "Run errors in SBA pytest"; fail_3=1; fi - if [ $zero_errors != 1 ]; then echo "Run errors in SBA pytest"; exit $EXIT_CODE_FAIL; fi - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures without non-BE tag encountered"; fail_3=1; fi - if [ $exit_code -eq 1 ] && [ $non_be_flag == 1 ]; then echo "pytest run had failures with non-BE tag encountered"; expected_nonbe_3=1; fi - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures without non-BE tag encountered"; exit $EXIT_CODE_FAIL; fi - if [ $exit_code -eq 1 ] && [ $non_be_flag == 1 ]; then echo "pytest run had failures with non-BE tag encountered"; exit $EXIT_CODE_NON_BE; fi # Check results from all three tests - if [ $fail_1 -eq 1 ] || [ $fail_2 -eq 1 ] || [ $fail_3 -eq 1 ]; then exit $EXIT_CODE_FAIL; fi - if [ $expected_nonbe_1 -eq 1 ] || [ $expected_nonbe_2 -eq 1 ] || [ $expected_nonbe_3 -eq 1 ]; then exit $EXIT_CODE_NON_BE; fi - exit 0 allow_failure: Loading @@ -352,16 +368,58 @@ self-test-on-merge-request: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" when: always paths: - test_output.txt - test_output_evs.txt - scripts/test/logs/ - scripts/ref/logs/ - report-junit.xml expose_as: 'Self test results' expose_as: 'SBA test results' reports: junit: report-junit.xml # compare bit exactness between target and source branch -- EVS self_test cases self-test-evs-on-merge-request: extends: - .test-job-linux - .rules-merge-request stage: compare needs: [ "build-codec-linux-cmake", "codec-smoke-test" ] timeout: "10 minutes" script: - *print-common-info - *merge_request_comparison_setup ### run selftest - ls -altr scripts/testv - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test scripts/config/self_test_evs.prm | tee test_output_evs.txt ### analyse test output # some helper variables - "|| true" to prevent failures from grep not finding anything - evs_non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[evs[ -]*non[ -]*be\]") || true - run_errors=$(cat test_output_evs.txt | grep -c "test conditions had run errors") || true - bitexact_evs=$(cat test_output_evs.txt | grep -c "All [0-9]* tests are bitexact") || true - EXIT_CODE_NON_BE=123 - EXIT_CODE_FAIL=1 # check for crashes during the test, if any happened, fail the test - if [ $run_errors != 0 ] ; then echo "Run errors in self_test_evs.py"; exit $EXIT_CODE_FAIL; fi # check for non bitexact EVS output - if [ $bitexact_evs == 0 ] && [ $evs_non_be_flag == 0 ] ; then echo "Non-bitexact EVS cases without EVS-non-BE tag encountered"; exit $EXIT_CODE_FAIL; fi - if [ $bitexact_evs == 0 ] && [ $evs_non_be_flag != 0 ] ; then echo "Non-bitexact EVS cases with EVS-non-BE tag encountered"; exit $EXIT_CODE_NON_BE; fi - exit 0 allow_failure: exit_codes: - 123 artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" when: always paths: - test_output_evs.txt - scripts/test/logs/ - scripts/ref/logs/ expose_as: 'Self test EVS results' # --------------------------------------------------------------- # Test jobs for main branch # --------------------------------------------------------------- Loading Loading
.gitlab-ci.yml +124 −66 Original line number Diff line number Diff line Loading @@ -41,6 +41,43 @@ stages: .get-previous-merge-commit-sha: &get-previous-merge-commit-sha - previous_merge_commit=$(git --no-pager log --merges HEAD~1 -n 1 --pretty=format:%H) .merge_request_comparison_setup: &merge_request_comparison_setup ### build test binaries, initial clean for paranoia reasons - make clean - mkdir build - cd build - cmake .. - make -j - mv IVAS_cod ../IVAS_cod_test - mv IVAS_dec ../IVAS_dec_test - cd .. - rm -rf build/* ### store the current commit hash - source_branch_commit_sha=$(git rev-parse HEAD) ### checkout version to compare against # first delete local target branch to avoid conflicts when branch is cached and there are merge conflicts during fetching # depending on chaching, the branch may not be there, so prevent failure of this command -> should maybe be done smarter later - git branch -D $CI_MERGE_REQUEST_TARGET_BRANCH_NAME || true # needed when depth is lower than the number of commits in the branch - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:$CI_MERGE_REQUEST_TARGET_BRANCH_NAME ### compare to last target branch commit before pipeline was created - target_commit=$(git log $CI_MERGE_REQUEST_TARGET_BRANCH_NAME -1 --oneline --before=${CI_PIPELINE_CREATED_AT} --format=%H) - git checkout $target_commit ### build reference binaries - cd build - cmake .. - make -j - mv IVAS_cod ../IVAS_cod_ref - mv IVAS_dec ../IVAS_dec_ref - cd .. ### re-checkout the commit from the source branch to have up-to-date self_test.py and scripts/testv (and actually everything) - git checkout $source_branch_commit_sha # --------------------------------------------------------------- # Job templates Loading Loading @@ -252,8 +289,8 @@ asan-on-merge-request-linux: expose_as: 'Asan selftest results' # compare bit exactness between target and source branch self-test-on-merge-request: # compare bit exactness between target and source branch -- IVAS self_test cases self-test-ivas-on-merge-request: extends: - .test-job-linux - .rules-merge-request Loading @@ -262,87 +299,66 @@ self-test-on-merge-request: timeout: "10 minutes" script: - *print-common-info ### build test binaries, initial clean for paranoia reasons - make clean - mkdir build - cd build - cmake .. - make -j - mv IVAS_cod ../IVAS_cod_test - mv IVAS_dec ../IVAS_dec_test - cd .. - rm -rf build/* ### store the current commit hash - source_branch_commit_sha=$(git rev-parse HEAD) ### checkout version to compare against # first delete local target branch to avoid conflicts when branch is cached and there are merge conflicts during fetching # depending on chaching, the branch may not be there, so prevent failure of this command -> should maybe be done smarter later - git branch -D $CI_MERGE_REQUEST_TARGET_BRANCH_NAME || true # needed when depth is lower than the number of commits in the branch - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:$CI_MERGE_REQUEST_TARGET_BRANCH_NAME ### compare to last target branch commit before pipeline was created - target_commit=$(git log $CI_MERGE_REQUEST_TARGET_BRANCH_NAME -1 --oneline --before=${CI_PIPELINE_CREATED_AT} --format=%H) - git checkout $target_commit ### build reference binaries - cd build - cmake .. - make -j - mv IVAS_cod ../IVAS_cod_ref - mv IVAS_dec ../IVAS_dec_ref - cd .. ### re-checkout the commit from the source branch to have up-to-date self_test.py and scripts/testv (and actually everything) - git checkout $source_branch_commit_sha - *merge_request_comparison_setup ### run selftest - ls -altr scripts/testv - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test | tee test_output.txt - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test scripts/config/self_test_evs.prm | tee test_output_evs.txt ### analyse test output # some helper variables - "|| true" to prevent failures from grep not finding anything - evs_non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[evs[ -]*non[ -]*be\]") || true - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[non[ -]*be\]") || true - run_errors=$(cat test_output.txt test_output_evs.txt | grep -c "test conditions had run errors") || true - bitexact=$(cat test_output.txt | grep -c "All [0-9]* tests are bitexact") || true - bitexact_evs=$(cat test_output_evs.txt | grep -c "All [0-9]* tests are bitexact") || true - EXIT_CODE_NON_BE=123 - EXIT_CODE_FAIL=1 - expected_nonbe_1=0 - expected_nonbe_2=0 - expected_nonbe_3=0 - fail_1=0 - fail_2=0 - fail_3=0 # check for crashes during the test, if any happened, fail the test - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; fail_1=1; fi - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; exit $EXIT_CODE_FAIL; fi # check for non bitexact output and store exit code to also always run the SBA pytest - if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; fail_1=1; fi - if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; expected_nonbe_1=1; fi - if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; exit $EXIT_CODE_FAIL; fi - if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; exit $EXIT_CODE_NON_BE; fi - exit 0 # check for non bitexact EVS output - if [ $bitexact_evs == 0 ] && [ $evs_non_be_flag == 0 ] ; then echo "Non-bitexact EVS cases without EVS-non-BE tag encountered"; fail_2=1; fi - if [ $bitexact_evs == 0 ] && [ $evs_non_be_flag != 0 ] ; then echo "Non-bitexact EVS cases with EVS-non-BE tag encountered"; expected_nonbe_2=1; fi allow_failure: exit_codes: - 123 artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" when: always paths: - test_output.txt - test_output_evs.txt - scripts/test/logs/ - scripts/ref/logs/ expose_as: 'Self test IVAS results' # compare bit exactness between target and source branch -- SBA self_test cases self-test-sba-on-merge-request: extends: - .test-job-linux - .rules-merge-request stage: compare needs: [ "build-codec-linux-cmake", "codec-smoke-test" ] timeout: "10 minutes" script: - *print-common-info - *merge_request_comparison_setup ### run SBA pytest - EXIT_CODE_NON_BE=123 - EXIT_CODE_FAIL=1 - exit_code=0 - python3 ./scripts/ivas_pytests/self_test_b.py --encref IVAS_cod_ref --decref IVAS_dec_ref --encdut IVAS_cod_test --decdut IVAS_dec_test || exit_code=$? - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true - if [ $zero_errors != 1 ]; then echo "Run errors in SBA pytest"; fail_3=1; fi - if [ $zero_errors != 1 ]; then echo "Run errors in SBA pytest"; exit $EXIT_CODE_FAIL; fi - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures without non-BE tag encountered"; fail_3=1; fi - if [ $exit_code -eq 1 ] && [ $non_be_flag == 1 ]; then echo "pytest run had failures with non-BE tag encountered"; expected_nonbe_3=1; fi - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures without non-BE tag encountered"; exit $EXIT_CODE_FAIL; fi - if [ $exit_code -eq 1 ] && [ $non_be_flag == 1 ]; then echo "pytest run had failures with non-BE tag encountered"; exit $EXIT_CODE_NON_BE; fi # Check results from all three tests - if [ $fail_1 -eq 1 ] || [ $fail_2 -eq 1 ] || [ $fail_3 -eq 1 ]; then exit $EXIT_CODE_FAIL; fi - if [ $expected_nonbe_1 -eq 1 ] || [ $expected_nonbe_2 -eq 1 ] || [ $expected_nonbe_3 -eq 1 ]; then exit $EXIT_CODE_NON_BE; fi - exit 0 allow_failure: Loading @@ -352,16 +368,58 @@ self-test-on-merge-request: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" when: always paths: - test_output.txt - test_output_evs.txt - scripts/test/logs/ - scripts/ref/logs/ - report-junit.xml expose_as: 'Self test results' expose_as: 'SBA test results' reports: junit: report-junit.xml # compare bit exactness between target and source branch -- EVS self_test cases self-test-evs-on-merge-request: extends: - .test-job-linux - .rules-merge-request stage: compare needs: [ "build-codec-linux-cmake", "codec-smoke-test" ] timeout: "10 minutes" script: - *print-common-info - *merge_request_comparison_setup ### run selftest - ls -altr scripts/testv - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test scripts/config/self_test_evs.prm | tee test_output_evs.txt ### analyse test output # some helper variables - "|| true" to prevent failures from grep not finding anything - evs_non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[evs[ -]*non[ -]*be\]") || true - run_errors=$(cat test_output_evs.txt | grep -c "test conditions had run errors") || true - bitexact_evs=$(cat test_output_evs.txt | grep -c "All [0-9]* tests are bitexact") || true - EXIT_CODE_NON_BE=123 - EXIT_CODE_FAIL=1 # check for crashes during the test, if any happened, fail the test - if [ $run_errors != 0 ] ; then echo "Run errors in self_test_evs.py"; exit $EXIT_CODE_FAIL; fi # check for non bitexact EVS output - if [ $bitexact_evs == 0 ] && [ $evs_non_be_flag == 0 ] ; then echo "Non-bitexact EVS cases without EVS-non-BE tag encountered"; exit $EXIT_CODE_FAIL; fi - if [ $bitexact_evs == 0 ] && [ $evs_non_be_flag != 0 ] ; then echo "Non-bitexact EVS cases with EVS-non-BE tag encountered"; exit $EXIT_CODE_NON_BE; fi - exit 0 allow_failure: exit_codes: - 123 artifacts: name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results" when: always paths: - test_output_evs.txt - scripts/test/logs/ - scripts/ref/logs/ expose_as: 'Self test EVS results' # --------------------------------------------------------------- # Test jobs for main branch # --------------------------------------------------------------- Loading
