Merge branch 'main' into add_evs_be_test_to_ci (37170004) · Commits · IVAS Codec Public Collaboration / IVAS Codec

.gitlab-ci.yml

+226 −49

Original line number	Diff line number	Diff line
		@@ -3,13 +3,16 @@ variables:
		BUILD_OUTPUT: "build_output.txt"
		EVS_BE_TEST_DIR: "/usr/local/be_2_evs_test"

		# prevent running two pipelines on pushes to merge request branches

		# This sets when pipelines are created. Jobs have more specific rules to restrict them.
		workflow:
		rules:
		# see https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines
		- if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push"
		when: never
		- when: always
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event' # Runs for merge requests
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main
		- if: $CI_PIPELINE_SOURCE == 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Scheduled in main


		stages:
		@@ -18,22 +21,63 @@ stages:
		- test
		- compare

		# ---------------------------------------------------------------
		# Generic script anchors
		# ---------------------------------------------------------------

		# These can be used later on to do common tasks

		# Prints useful information for every job and should be used at the beginning of each job
		.print-common-info: &print-common-info
		- \|
		echo "Printing common information for build job."
		echo "Current job is run on commit $CI_COMMIT_SHA"
		echo "Commit time was $CI_COMMIT_TIMESTAMP"
		date \| xargs echo "System time is"

		.get-previous-merge-commit-sha: &get-previous-merge-commit-sha
		- previous_merge_commit=$(git --no-pager log --merges HEAD~1 -n 1 --pretty=format:%H)


		# template for all test jobs
		.test-job:
		# ---------------------------------------------------------------
		# Job templates
		# ---------------------------------------------------------------

		# When designing templates, try not to use too much inheritance and
		# if multiple templates and extended on, remember that on conflict,
		# latest overwrites the parameter.

		# templates for rules
		.rules-basis:
		rules:
		- if: $MIRROR_ACCESS_TOKEN # Don't run in the mirror update pipeline (only then MIRROR_ACCESS_TOKEN is defined)
		when: never
		- when: on_success

		# template test job on linux
		.rules-merge-request:
		extends: .rules-basis
		rules:
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
		- if: $CI_PIPELINE_SOURCE == 'push'
		when: never

		.rules-main-push:
		extends: .rules-basis
		rules:
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

		.rules-main-scheduled:
		extends: .rules-basis
		rules:
		- if: $CI_PIPELINE_SOURCE == 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH


		# templates to define stages and platforms
		.test-job-linux:
		extends: .test-job
		tags:
		- ivas-linux

		.build-job-linux:
		extends: .test-job
		stage: build
		timeout: "2 minutes"
		tags:
		@@ -56,47 +100,56 @@ stages:
		- 123


		# ---------------------------------------------------------------
		# Build jobs
		# ---------------------------------------------------------------

		build-codec-linux-make:
		extends: .build-job-with-check-for-warnings
		rules:
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
		extends:
		- .build-job-with-check-for-warnings
		- .rules-basis
		script:
		- *print-common-info
		- make -j 2>&1 \| tee $BUILD_OUTPUT
		# need to use the "\|\| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<
		- ci/check_for_warnings.py $BUILD_OUTPUT \|\| exit $?

		build-unittests-linux:
		extends: .build-job-with-check-for-warnings
		rules:
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
		extends:
		- .build-job-with-check-for-warnings
		- .rules-basis
		script:
		- *print-common-info
		- make unittests -j 2>&1 \| tee $BUILD_OUTPUT
		# need to use the "\|\| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<
		- ci/check_for_warnings.py $BUILD_OUTPUT \|\| exit $?

		build-prerenderer-linux:
		extends: .build-job-with-check-for-warnings
		rules:
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
		extends:
		- .build-job-with-check-for-warnings
		- .rules-basis
		script:
		- *print-common-info
		- make -C scripts/prerenderer -j 2>&1 \| tee $BUILD_OUTPUT
		# need to use the "\|\| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<
		- ci/check_for_warnings.py $BUILD_OUTPUT \|\| exit $?

		build-td-object-renderer-standalone-linux:
		extends: .build-job-with-check-for-warnings
		rules:
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
		extends:
		- .build-job-with-check-for-warnings
		- .rules-basis
		script:
		- *print-common-info
		- make -C scripts/td_object_renderer/object_renderer_standalone -j 2>&1 \| tee $BUILD_OUTPUT
		# need to use the "\|\| exit $?" suffix to get the allowed_failure return code, otherwise the job fails with code 1...<
		- ci/check_for_warnings.py $BUILD_OUTPUT \|\| exit $?

		build-codec-linux-cmake:
		extends: .build-job-with-check-for-warnings
		rules:
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
		extends:
		- .build-job-with-check-for-warnings
		- .rules-basis
		script:
		- *print-common-info
		- mkdir build
		- cd build
		- cmake ..
		@@ -106,47 +159,57 @@ build-codec-linux-cmake:
		- ci/check_for_warnings.py $BUILD_OUTPUT \|\| exit $?

		build-codec-instrumented-linux:
		extends: .build-job-linux
		rules:
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
		extends:
		- .build-job-linux
		- .rules-basis
		script:
		- *print-common-info
		- bash ci/build_codec_instrumented_linux.sh

		# make sure that the codec builds with msan, asan and usan
		build-codec-sanitizers-linux:
		extends: .build-job-linux
		rules:
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
		extends:
		- .build-job-linux
		- .rules-basis
		script:
		- *print-common-info
		- bash ci/build_codec_sanitizers_linux.sh


		# ---------------------------------------------------------------
		# Test jobs for merge requests
		# ---------------------------------------------------------------

		# test that runs all modes with 1s input signals
		codec-smoke-test:
		extends: .test-job-linux-needs-testv-dir
		# temporarily restrict this job to the only runner which (so far) seems to runit without problems
		extends:
		- .test-job-linux-needs-testv-dir
		- .rules-merge-request
		timeout: "5 minutes"
		rules:
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
		stage: test
		needs: [ "build-codec-linux-cmake" ]
		script:
		- *print-common-info
		- bash ci/smoke_test.sh
		### analyze for failures
		- if cat smoke_test_output.txt \| grep -c "failed"; then echo "Smoke test without PLC failed"; exit 1; fi
		- if cat smoke_test_output_plc.txt \| grep -c "failed"; then echo "Smoke test with PLC failed"; exit 1; fi
		artifacts:
		name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"
		paths:
		- out/logs
		- out/logs/
		- smoke_test_output.txt
		- smoke_test_output_plc.txt
		expose_as: 'Smoke test results'


		# check bitexactness to EVS
		be-2-evs-linux:
		extends: .test-job-linux
		extends:
		- .test-job-linux
		- .rules-merge-request
		tags:
		- test-fhg-linux-runner1
		rules:
		- if: $CI_COMMIT_BRANCH == 'main'
		stage: test
		script:
		- mkdir build
		@@ -162,14 +225,17 @@ be-2-evs-linux:

		- cd evs_be_test
		- python3 ../ci/run_evs_be_test.py


		# code selftest testvectors with memory-sanitizer binaries
		msan-on-merge-request-linux:
		extends: .test-job-linux
		extends:
		- .test-job-linux
		- .rules-merge-request
		stage: test
		needs: [ "build-codec-sanitizers-linux" ]
		rules:
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
		script:
		- *print-common-info
		- make clean
		- make -j CLANG=1
		- python3 scripts/self_test.py --create \| tee test_output.txt
		@@ -179,12 +245,13 @@ msan-on-merge-request-linux:

		# code selftest testvectors with address-sanitizer binaries
		asan-on-merge-request-linux:
		extends: .test-job-linux
		extends:
		- .test-job-linux
		- .rules-merge-request
		stage: test
		needs: [ "build-codec-sanitizers-linux" ]
		rules:
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
		script:
		- *print-common-info
		- make clean
		- make -j CLANG=2
		- python3 scripts/self_test.py --create \| tee test_output.txt
		@@ -194,15 +261,14 @@ asan-on-merge-request-linux:

		# compare bit exactness between target and source branch
		self-test-on-merge-request:
		extends: .test-job-linux
		tags:
		- test-fhg-linux-runner1
		extends:
		- .test-job-linux
		- .rules-merge-request
		stage: compare
		needs: [ "build-codec-linux-cmake", "codec-smoke-test" ]
		rules:
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
		timeout: "10 minutes"
		script:
		- *print-common-info
		### build test binaries, initial clean for paranoia reasons
		- make clean
		- mkdir build
		@@ -274,16 +340,127 @@ self-test-on-merge-request:
		exit_codes:
		- 123
		artifacts:
		name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"
		when: always
		paths:
		- test_output.txt
		- scripts/test/logs
		- scripts/ref/logs
		- scripts/test/logs/
		- scripts/ref/logs/
		- report-junit.xml
		expose_as: 'Self test results'
		reports:
		junit: report-junit.xml


		# ---------------------------------------------------------------
		# Test jobs for main branch
		# ---------------------------------------------------------------

		codec-comparison-on-main-push:
		extends:
		- .test-job-linux
		- .rules-main-push
		stage: compare
		needs: [ "build-codec-linux-cmake" ]
		timeout: "30 minutes" # To be revisited
		script:
		- *print-common-info
		- latest_commit=$(git rev-parse HEAD) # Latest commit
		- *get-previous-merge-commit-sha # Stored in previous_merge_commit shell variable now
		- echo "Comparing changes from $previous_merge_commit to $latest_commit"
		- git --no-pager diff --stat $previous_merge_commit..$latest_commit

		# Rest is more or less placeholder adapted from MR self test. This should be replaced with more complex tests.

		### build test binaries, initial clean for paranoia reasons
		- make clean
		- mkdir build
		- cd build
		- cmake ..
		- make -j
		- mv IVAS_cod ../IVAS_cod_test
		- mv IVAS_dec ../IVAS_dec_test
		- cd ..
		- rm -rf build/*

		### compare to the previous merge commit in the main branch
		- git fetch origin main
		- git checkout $previous_merge_commit

		### build reference binaries
		- cd build
		- cmake ..
		- make -j
		- mv IVAS_cod ../IVAS_cod_ref
		- mv IVAS_dec ../IVAS_dec_ref
		- cd ..

		### re-checkout the latest commit in the main branch
		- git checkout $latest_commit

		### run selftest
		- ls -altr scripts/testv
		- python3 ./scripts/self_test.py --encref IVAS_cod_ref --decref IVAS_dec_ref --enctest IVAS_cod_test --dectest IVAS_dec_test \| tee test_output.txt

		### analyse test output

		# some helper variables - "\|\| true" to prevent failures from grep not finding anything
		- non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE \| grep -c --ignore-case "\[non[ -]*be\]") \|\| true
		- run_errors=$(cat test_output.txt \| grep -c "test conditions had run errors") \|\| true
		- bitexact=$(cat test_output.txt \| grep -c "All [0-9]* tests are bitexact") \|\| true
		- EXIT_CODE_NON_BE=123
		- EXIT_CODE_FAIL=1

		- selftest_exit_code=0

		# check for crashes during the test, if any happened, fail the test
		- if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py"; exit $EXIT_CODE_FAIL; fi

		# check for non bitexact output and store exit code to also always run the SBA pytest
		- if [ $bitexact == 0 ] && [ $non_be_flag == 0 ] ; then echo "Non-bitexact cases without non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_FAIL; fi
		- if [ $bitexact == 0 ] && [ $non_be_flag != 0 ]; then echo "Non-bitexact cases with non-BE tag encountered"; selftest_exit_code=$EXIT_CODE_NON_BE; fi

		### run SBA pytest
		- exit_code=0
		- python3 ./scripts/ivas_pytests/self_test_b.py --encref IVAS_cod_ref --decref IVAS_dec_ref --encdut IVAS_cod_test --decdut IVAS_dec_test \|\| exit_code=$?
		- if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi
		- zero_errors=$(cat report-junit.xml \| grep -c 'errors="0"') \|\| true
		- if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi
		- if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi;
		# return exit code from selftest if everything went well with the pytest run
		- exit $selftest_exit_code
		allow_failure:
		exit_codes:
		- 123
		artifacts:
		name: "main-push--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"
		when: always
		paths:
		- test_output.txt
		- scripts/test/logs/
		- scripts/ref/logs/
		- report-junit.xml
		expose_as: 'Results of comparison to previous merge commit'
		reports:
		junit: report-junit.xml


		sanitizer-test-on-main-scheduled:
		extends: .test-job-linux-needs-testv-dir
		stage: test
		rules:
		# only run in scheduled pipeline that passes this env var
		- if: $SANITIZER_TEST_IN_FMT
		script:
		- *print-common-info
		- echo "Running scheduled sanitizer"
		# - python3 ci/run_scheduled_sanitizer_test.py $SANITIZER_TEST_IN_FMT $SANITIZER_TEST_OUT_FMTS


		# ---------------------------------------------------------------
		# Other jobs
		# ---------------------------------------------------------------

		# Pull state of a branch on 3GPP repo, push to a mirror repo.
		pull-from-3gpp-forge:
		stage: maintenance

.gitlab/issue_templates/bug_report.md

0 → 100644

+19 −0

Original line number	Diff line number	Diff line
		### Basic info

		- Commit SHA:
		- Platform:

		### Bug description


		### Ways to reproduce

		(Clear steps or refer to a failing automated test, e.g. with a pipeline link)



		<!--- Below are labels that will be added but are not shown in description. This is a template to help fill them.
		Add further information to the first row and remove and add labels as necessary. -->

		/label ~Priority:: ~Company: ~Subpart:
		/label ~Type:Bug ~Status::ToDo

.gitlab/issue_templates/generic_issue.md

0 → 100644

+7 −0

Original line number	Diff line number	Diff line
		(Generic issue template)

		<!--- Below are labels that will be added but are not shown in description. This is a template to help fill them.
		Add further information to the first row and remove and add labels as necessary. -->

		/label ~Priority:: ~Company: ~Subpart: ~Type:
		/label ~Status::ToDo

.gitlab/merge_request_templates/generic_merge_request.md

0 → 100644

+12 −0

Original line number	Diff line number	Diff line
		<!--- Basic information that is useful -->
		- Related issues:
		- Requested reviewers:

		### Description


		<!--- Labels that are added. Fill as necessary. -->
		/label ~Priority:: ~Company: ~Subpart: ~Type: ~Status::

		<!--- Alternatively to labels, you can use "/copy_metadata #number" to get labels from an issue -->

.gitlab/merge_request_templates/maintenance_task_mr.md

0 → 100644

+9 −0

Original line number	Diff line number	Diff line
		- Requested reviewers:

		### Description


		<!--- Labels that are added. Fill as necessary. -->
		/label ~Maintenance:
		/label ~Status::
		/label ~Type: