Adds self test for main branch push comparing to the previous merge commit.... (6d2ee847) · Commits · IVAS Codec Public Collaboration / IVAS Codec

.gitlab-ci.yml

+106 −3

Original line number	Diff line number	Diff line
		@@ -20,6 +20,23 @@ stages:
		- test
		- compare

		# ---------------------------------------------------------------
		# Generic script anchors
		# ---------------------------------------------------------------

		# These can be used later on to do common tasks

		# Prints useful information for every job and should be used at the beginning of each job
		.print-common-info: &print-common-info
		- \|
		echo "Printing common information for build job."
		echo "Current job is run on commit $CI_COMMIT_SHA"
		echo "Commit time was $CI_COMMIT_TIMESTAMP"
		date \| xargs echo "System time is"

		.get-previous-merge-commit-sha: &get-previous-merge-commit-sha
		- previous_merge_commit=$(git --no-pager log --merges HEAD~1 -n 1 --pretty=format:%H)


		# ---------------------------------------------------------------
		# Job templates
		@@ -91,6 +108,7 @@ build-codec-linux-make:
		- .build-job-with-check-for-warnings
		- .rules-basis
		script:
		- *print-common-info
		- make -j 2>&1 \| tee $BUILD_OUTPUT
		# need to use the "\|\| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<
		- ci/check_for_warnings.py $BUILD_OUTPUT \|\| exit $?
		@@ -100,6 +118,7 @@ build-unittests-linux:
		- .build-job-with-check-for-warnings
		- .rules-basis
		script:
		- *print-common-info
		- make unittests -j 2>&1 \| tee $BUILD_OUTPUT
		# need to use the "\|\| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<
		- ci/check_for_warnings.py $BUILD_OUTPUT \|\| exit $?
		@@ -109,6 +128,7 @@ build-prerenderer-linux:
		- .build-job-with-check-for-warnings
		- .rules-basis
		script:
		- *print-common-info
		- make -C scripts/prerenderer -j 2>&1 \| tee $BUILD_OUTPUT
		# need to use the "\|\| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<
		- ci/check_for_warnings.py $BUILD_OUTPUT \|\| exit $?
		@@ -118,6 +138,7 @@ build-td-object-renderer-standalone-linux:
		- .build-job-with-check-for-warnings
		- .rules-basis
		script:
		- *print-common-info
		- make -C scripts/td_object_renderer/object_renderer_standalone -j 2>&1 \| tee $BUILD_OUTPUT
		# need to use the "\|\| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<
		- ci/check_for_warnings.py $BUILD_OUTPUT \|\| exit $?
		@@ -127,6 +148,7 @@ build-codec-linux-cmake:
		- .build-job-with-check-for-warnings
		- .rules-basis
		script:
		- *print-common-info
		- mkdir build
		- cd build
		- cmake ..
		@@ -140,6 +162,7 @@ build-codec-instrumented-linux:
		- .build-job-linux
		- .rules-basis
		script:
		- *print-common-info
		- bash ci/build_codec_instrumented_linux.sh

		# make sure that the codec builds with msan, asan and usan
		@@ -148,6 +171,7 @@ build-codec-sanitizers-linux:
		- .build-job-linux
		- .rules-basis
		script:
		- *print-common-info
		- bash ci/build_codec_sanitizers_linux.sh


		@@ -164,6 +188,7 @@ codec-smoke-test:
		stage: test
		needs: [ "build-codec-linux-cmake" ]
		script:
		- *print-common-info
		- bash ci/smoke_test.sh
		### analyze for failures
		- if cat smoke_test_output.txt \| grep -c "failed"; then echo "Smoke test without PLC failed"; exit 1; fi
		@@ -181,6 +206,7 @@ msan-on-merge-request-linux:
		stage: test
		needs: [ "build-codec-sanitizers-linux" ]
		script:
		- *print-common-info
		- make clean
		- make -j CLANG=1
		- python3 scripts/self_test.py --create \| tee test_output.txt
		@@ -196,6 +222,7 @@ asan-on-merge-request-linux:
		stage: test
		needs: [ "build-codec-sanitizers-linux" ]
		script:
		- *print-common-info
		- make clean
		- make -j CLANG=2
		- python3 scripts/self_test.py --create \| tee test_output.txt
		@@ -212,6 +239,7 @@ self-test-on-merge-request:
		needs: [ "build-codec-linux-cmake", "codec-smoke-test" ]
		timeout: "10 minutes"
		script:
		- *print-common-info
		### build test binaries, initial clean for paranoia reasons
		- make clean
		- mkdir build
		@@ -305,9 +333,83 @@ codec-comparison-on-main-push:
		needs: [ "build-codec-linux-cmake" ]
		timeout: "30 minutes" # To be revisited
		script:
		- echo "Linux, run longer comparison to previous main push"
		- git rev-parse HEAD # Current
		- git rev-parse HEAD~1 # Previous
		- *print-common-info
		- latest_commit=$(git rev-parse HEAD) # Latest commit
		- *get-previous-merge-commit-sha # Stored in previous_merge_commit shell variable now
		- echo "Comparing changes from $previous_merge_commit to $latest_commit"
		- git --no-pager diff --stat $previous_merge_commit..$latest_commit

		# Rest is more or less placeholder adapted from MR self test. This should be replaced with more complex tests.

		### build test binaries, initial clean for paranoia reasons
		- make clean
		- mkdir build
		- cd build
		- cmake ..
		- make -j
		- mv IVAS_cod ../IVAS_cod_test
		- mv IVAS_dec ../IVAS_dec_test
		- cd ..
		- rm -rf build/*

		### compare to the previous merge commit in the main branch
		- git fetch origin main
		- git checkout $previous_merge_commit

		### build reference binaries
		- cd build
		- cmake ..
		- make -j
		- mv IVAS_cod ../IVAS_cod_ref
		- mv IVAS_dec ../IVAS_dec_ref
		- cd ..

		### re-checkout the latest commit in the main branch
		- git checkout $latest_commit

		### run selftest
		- ls -altr scripts/testv
		- python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test \| tee test_output.txt

		### analyse test output

		# some helper variables - "\|\| true" to prevent failures from grep not finding anything
		- non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE \| grep -c --ignore-case "\[non[ -]*be\]") \|\| true
		- run_errors=$(cat test_output.txt \| grep -c "test conditions had run errors") \|\| true
		- bitexact=$(cat test_output.txt \| grep -c "All [0-9]* tests are bitexact") \|\| true
		- EXIT_CODE_NON_BE=123
		- EXIT_CODE_FAIL=1

		- selftest_exit_code=0

		# check for crashes during the test, if any happened, fail the test
		- if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; exit $EXIT_CODE_FAIL; fi

		# check for non bitexact output and store exit code to also always run the SBA pytest
		- if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_FAIL; fi
		- if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_NON_BE; fi

		### run SBA pytest
		- exit_code=0
		- python3 ./scripts/ivas_pytests/self_test_b.py --encref IVAS_cod_ref --decref IVAS_dec_ref --encdut IVAS_cod_test --decdut IVAS_dec_test \|\| exit_code=$?
		- if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi
		- zero_errors=$(cat report-junit.xml \| grep -c 'errors="0"') \|\| true
		- if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi
		- if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi;
		# return exit code from selftest if everything went well with the pytest run
		- exit $selftest_exit_code
		allow_failure:
		exit_codes:
		- 123
		artifacts:
		when: always
		paths:
		- test_output.txt
		- scripts/test/logs
		- scripts/ref/logs
		- report-junit.xml
		reports:
		junit: report-junit.xml


		sanitizer-test-on-main-scheduled:
		@@ -317,6 +419,7 @@ sanitizer-test-on-main-scheduled:
		# only run in scheduled pipeline that passes this env var
		- if: $SANITIZER_TEST_IN_FMT
		script:
		- *print-common-info
		- echo "Running scheduled sanitizer"
		# - python3 ci/run_scheduled_sanitizer_test.py $SANITIZER_TEST_IN_FMT $SANITIZER_TEST_OUT_FMTS

Admin message