diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ef9a4660538a6c09f363ae0bbabf942aed798856..b63e30472aea589b1683245d4028fc41e608626a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,13 +2,16 @@ variables: TESTV_DIR: "/usr/local/testv" BUILD_OUTPUT: "build_output.txt" -# prevent running two pipelines on pushes to merge request branches + +# This sets when pipelines are created. Jobs have more specific rules to restrict them. workflow: rules: # see https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" when: never - - when: always + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' # Runs for merge requests + - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main + - if: $CI_PIPELINE_SOURCE == 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Scheduled in main stages: @@ -17,22 +20,63 @@ stages: - test - compare +# --------------------------------------------------------------- +# Generic script anchors +# --------------------------------------------------------------- + +# These can be used later on to do common tasks + +# Prints useful information for every job and should be used at the beginning of each job +.print-common-info: &print-common-info + - | + echo "Printing common information for build job." + echo "Current job is run on commit $CI_COMMIT_SHA" + echo "Commit time was $CI_COMMIT_TIMESTAMP" + date | xargs echo "System time is" + +.get-previous-merge-commit-sha: &get-previous-merge-commit-sha + - previous_merge_commit=$(git --no-pager log --merges HEAD~1 -n 1 --pretty=format:%H) -# template for all test jobs -.test-job: + +# --------------------------------------------------------------- +# Job templates +# --------------------------------------------------------------- + +# When designing templates, try not to use too much inheritance and +# if multiple templates and extended on, remember that on conflict, +# latest overwrites the parameter. + +# templates for rules +.rules-basis: rules: - if: $MIRROR_ACCESS_TOKEN # Don't run in the mirror update pipeline (only then MIRROR_ACCESS_TOKEN is defined) when: never - when: on_success -# template test job on linux +.rules-merge-request: + extends: .rules-basis + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + - if: $CI_PIPELINE_SOURCE == 'push' + when: never + +.rules-main-push: + extends: .rules-basis + rules: + - if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + +.rules-main-scheduled: + extends: .rules-basis + rules: + - if: $CI_PIPELINE_SOURCE == 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + + +# templates to define stages and platforms .test-job-linux: - extends: .test-job tags: - ivas-linux .build-job-linux: - extends: .test-job stage: build timeout: "2 minutes" tags: @@ -55,47 +99,56 @@ stages: - 123 +# --------------------------------------------------------------- +# Build jobs +# --------------------------------------------------------------- + build-codec-linux-make: - extends: .build-job-with-check-for-warnings - rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + extends: + - .build-job-with-check-for-warnings + - .rules-basis script: + - *print-common-info - make -j 2>&1 | tee $BUILD_OUTPUT # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...< - ci/check_for_warnings.py $BUILD_OUTPUT || exit $? build-unittests-linux: - extends: .build-job-with-check-for-warnings - rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + extends: + - .build-job-with-check-for-warnings + - .rules-basis script: + - *print-common-info - make unittests -j 2>&1 | tee $BUILD_OUTPUT # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...< - ci/check_for_warnings.py $BUILD_OUTPUT || exit $? build-prerenderer-linux: - extends: .build-job-with-check-for-warnings - rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + extends: + - .build-job-with-check-for-warnings + - .rules-basis script: + - *print-common-info - make -C scripts/prerenderer -j 2>&1 | tee $BUILD_OUTPUT # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...< - ci/check_for_warnings.py $BUILD_OUTPUT || exit $? build-td-object-renderer-standalone-linux: - extends: .build-job-with-check-for-warnings - rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + extends: + - .build-job-with-check-for-warnings + - .rules-basis script: + - *print-common-info - make -C scripts/td_object_renderer/object_renderer_standalone -j 2>&1 | tee $BUILD_OUTPUT # need to use the "|| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...< - ci/check_for_warnings.py $BUILD_OUTPUT || exit $? build-codec-linux-cmake: - extends: .build-job-with-check-for-warnings - rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + extends: + - .build-job-with-check-for-warnings + - .rules-basis script: + - *print-common-info - mkdir build - cd build - cmake .. @@ -105,31 +158,37 @@ build-codec-linux-cmake: - ci/check_for_warnings.py $BUILD_OUTPUT || exit $? build-codec-instrumented-linux: - extends: .build-job-linux - rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + extends: + - .build-job-linux + - .rules-basis script: + - *print-common-info - bash ci/build_codec_instrumented_linux.sh # make sure that the codec builds with msan, asan and usan build-codec-sanitizers-linux: - extends: .build-job-linux - rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + extends: + - .build-job-linux + - .rules-basis script: + - *print-common-info - bash ci/build_codec_sanitizers_linux.sh +# --------------------------------------------------------------- +# Test jobs for merge requests +# --------------------------------------------------------------- + # test that runs all modes with 1s input signals codec-smoke-test: - extends: .test-job-linux-needs-testv-dir - # temporarily restrict this job to the only runner which (so far) seems to runit without problems + extends: + - .test-job-linux-needs-testv-dir + - .rules-merge-request timeout: "5 minutes" - rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' stage: test needs: [ "build-codec-linux-cmake" ] script: + - *print-common-info - bash ci/smoke_test.sh ### analyze for failures - if cat smoke_test_output.txt | grep -c "failed"; then echo "Smoke test without PLC failed"; exit 1; fi @@ -141,12 +200,13 @@ codec-smoke-test: # code selftest testvectors with memory-sanitizer binaries msan-on-merge-request-linux: - extends: .test-job-linux + extends: + - .test-job-linux + - .rules-merge-request stage: test needs: [ "build-codec-sanitizers-linux" ] - rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' script: + - *print-common-info - make clean - make -j CLANG=1 - python3 scripts/self_test.py --create | tee test_output.txt @@ -156,12 +216,13 @@ msan-on-merge-request-linux: # code selftest testvectors with address-sanitizer binaries asan-on-merge-request-linux: - extends: .test-job-linux + extends: + - .test-job-linux + - .rules-merge-request stage: test needs: [ "build-codec-sanitizers-linux" ] - rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' script: + - *print-common-info - make clean - make -j CLANG=2 - python3 scripts/self_test.py --create | tee test_output.txt @@ -171,13 +232,14 @@ asan-on-merge-request-linux: # compare bit exactness between target and source branch self-test-on-merge-request: - extends: .test-job-linux + extends: + - .test-job-linux + - .rules-merge-request stage: compare needs: [ "build-codec-linux-cmake", "codec-smoke-test" ] - rules: - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' timeout: "10 minutes" script: + - *print-common-info ### build test binaries, initial clean for paranoia reasons - make clean - mkdir build @@ -259,6 +321,113 @@ self-test-on-merge-request: junit: report-junit.xml +# --------------------------------------------------------------- +# Test jobs for main branch +# --------------------------------------------------------------- + +codec-comparison-on-main-push: + extends: + - .test-job-linux + - .rules-main-push + stage: compare + needs: [ "build-codec-linux-cmake" ] + timeout: "30 minutes" # To be revisited + script: + - *print-common-info + - latest_commit=$(git rev-parse HEAD) # Latest commit + - *get-previous-merge-commit-sha # Stored in previous_merge_commit shell variable now + - echo "Comparing changes from $previous_merge_commit to $latest_commit" + - git --no-pager diff --stat $previous_merge_commit..$latest_commit + + # Rest is more or less placeholder adapted from MR self test. This should be replaced with more complex tests. + + ### build test binaries, initial clean for paranoia reasons + - make clean + - mkdir build + - cd build + - cmake .. + - make -j + - mv IVAS_cod ../IVAS_cod_test + - mv IVAS_dec ../IVAS_dec_test + - cd .. + - rm -rf build/* + + ### compare to the previous merge commit in the main branch + - git fetch origin main + - git checkout $previous_merge_commit + + ### build reference binaries + - cd build + - cmake .. + - make -j + - mv IVAS_cod ../IVAS_cod_ref + - mv IVAS_dec ../IVAS_dec_ref + - cd .. + + ### re-checkout the latest commit in the main branch + - git checkout $latest_commit + + ### run selftest + - ls -altr scripts/testv + - python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test | tee test_output.txt + + ### analyse test output + + # some helper variables - "|| true" to prevent failures from grep not finding anything + - non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE | grep -c --ignore-case "\[non[ -]*be\]") || true + - run_errors=$(cat test_output.txt | grep -c "test conditions had run errors") || true + - bitexact=$(cat test_output.txt | grep -c "All [0-9]* tests are bitexact") || true + - EXIT_CODE_NON_BE=123 + - EXIT_CODE_FAIL=1 + + - selftest_exit_code=0 + + # check for crashes during the test, if any happened, fail the test + - if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; exit $EXIT_CODE_FAIL; fi + + # check for non bitexact output and store exit code to also always run the SBA pytest + - if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_FAIL; fi + - if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_NON_BE; fi + + ### run SBA pytest + - exit_code=0 + - python3 ./scripts/ivas_pytests/self_test_b.py --encref IVAS_cod_ref --decref IVAS_dec_ref --encdut IVAS_cod_test --decdut IVAS_dec_test || exit_code=$? + - if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi + - zero_errors=$(cat report-junit.xml | grep -c 'errors="0"') || true + - if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi + - if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi; + # return exit code from selftest if everything went well with the pytest run + - exit $selftest_exit_code + allow_failure: + exit_codes: + - 123 + artifacts: + when: always + paths: + - test_output.txt + - scripts/test/logs + - scripts/ref/logs + - report-junit.xml + reports: + junit: report-junit.xml + + +sanitizer-test-on-main-scheduled: + extends: .test-job-linux-needs-testv-dir + stage: test + rules: + # only run in scheduled pipeline that passes this env var + - if: $SANITIZER_TEST_IN_FMT + script: + - *print-common-info + - echo "Running scheduled sanitizer" + # - python3 ci/run_scheduled_sanitizer_test.py $SANITIZER_TEST_IN_FMT $SANITIZER_TEST_OUT_FMTS + + +# --------------------------------------------------------------- +# Other jobs +# --------------------------------------------------------------- + # Pull state of a branch on 3GPP repo, push to a mirror repo. pull-from-3gpp-forge: stage: maintenance