Update .gitlab-ci.yml (39159694) · Commits · IVAS Codec Public Collaboration / IVAS Codec

.gitlab-ci.yml

+47 −38

Original line number	Diff line number	Diff line
		@@ -141,10 +141,8 @@ stages:
		- source_branch_commit_sha=$(git rev-parse HEAD)

		### checkout version to compare against
		- *mr-fetch-target-branch
		- *mr-get-target-commit
		- git checkout $target_commit
		- echo "Building reference codec at commit $target_commit"
		- git checkout main
		- echo "Building reference codec at commit main"

		### build reference binaries
		- *disable-debugging-macro
		@@ -228,7 +226,7 @@ stages:
		- if [ $CLANG_NUM -eq 1 ]; then sanitizer_type="MemorySanitizer"; elif [ $CLANG_NUM -eq 2 ]; then sanitizer_type="AddressSanitizer"; elif [ $CLANG_NUM -eq 3 ]; then sanitizer_type="UndefinedBehaviorSanitizer"; else echo "Wrong CLANG_NUM $CLANG_NUM given!"; exit 1; fi

		# run encoder and decoder with 20ms renderer framesize first, use reference creation mode
		- python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec \|\| exit_code20=$?
		- python3 -m pytest tests/codec_be_on_mr_nonselection/test_param_file.py --param_file $SELF_TEST_PRM_FILE -v --update_ref 1 -m create_ref --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml --testcase_timeout=$testcase_timeout --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec \|\| exit_code20=$?
		# for 10ms and 5ms renderer framesize, we only need to run the decoder part as renderer framesize is a decoder-only option
		# set tolerance very high do ignore any BE differences due to the different renderer framesizes, those can appear due to the limiter being active
		# we are only interested in runtime errors from the sanitizers and ignore the diffs
		@@ -295,6 +293,8 @@ stages:
		needs: []
		tags:
		- ivas-linux
		rules:
		- if: 1==0

		.build-job-windows:
		stage: build
		@@ -441,22 +441,25 @@ check-compatibility-with-basop-reference-branch:
		- python3 tests/create_short_testvectors.py
		- exit_code1=0
		- exit_code2=0
		- python3 -m pytest tests/codec_be_on_mr_nonselection -v --update_ref 1 --html=report.html --self-contained-html --junit-xml=report-junit.xml \|\| exit_code1=$?
		- zero_errors=$(cat report-junit.xml \| grep -c 'errors="0"') \|\| true
		- if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi
		- python3 -m pytest tests/codec_be_on_mr_nonselection -v --update_ref 1 -m create_ref --html=report1.html --self-contained-html --junit-xml=report-junit1.xml \|\| exit_code1=$?
		- python3 -m pytest tests/codec_be_on_mr_nonselection -v --update_ref 1 -m create_ref_part2 --html=report2.html --self-contained-html --junit-xml=report-junit2.xml \|\| exit_code2=$?
		- zero_errors=$(cat report-junit1.xml report-junit2.xml \| grep -c 'errors="0"') \|\| true
		- if [ $zero_errors != 2 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi
		- exit 0
		artifacts:
		name: "check-float-reference--sha-$CI_COMMIT_SHORT_SHA--results"
		when: always
		expire_in: 1 month
		paths:
		- report-junit.xml
		- report.html

		- report-junit1.xml
		- report-junit2.xml
		- report1.html
		- report2.html
		expose_as: "check-float-reference results"
		reports:
		junit:
		- report-junit.xml
		- report-junit1.xml
		- report-junit2.xml

		# ---------------------------------------------------------------
		# Build jobs
		@@ -610,7 +613,8 @@ pytest-compare-20ms-and-5ms-rendering:
		- cp IVAS_cod IVAS_cod_ref
		- cp IVAS_dec IVAS_dec_ref
		# create references
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref_part2
		### run pytest
		- exit_code=0
		- exit_code5=0
		@@ -900,14 +904,15 @@ split-rendering-pytest-on-merge-request:
		ivas-pytest-on-merge-request:
		extends:
		- .test-job-linux-needs-testv-dir
		- .rules-merge-request
		tags:
		- test-ericsson-linux-sprinter-1
		rules:
		# only run in scheduled pipeline that passes this env vars
		- if: $TRIGGER_MR_PYTEST
		stage: compare
		needs: ["build-codec-linux-cmake", "codec-smoke-test"]
		timeout: "14 minutes"
		script:
		- *print-common-info
		- *get-commits-behind-count
		- *check-commits-behind-count-in-compare-jobs
		- *merge-request-comparison-setup-codec

		# some helper variables - "\|\| true" to prevent failures from grep not finding anything
		@@ -921,7 +926,8 @@ ivas-pytest-on-merge-request:

		### prepare pytest
		# create references
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref_part2

		### Run test using branch scripts and input
		- if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi
		@@ -978,11 +984,12 @@ ivas-interop-on-merge-request:
		- exit_code2=0
		# set timeout for individual testcase runs to 60 seconds
		- testcase_timeout=60
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout \|\| exit_code=$?
		- zero_failures=$(cat report-junit.xml \| grep -c 'failures="0"') \|\| true
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout \|\| exit_code=$?
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report2.html --self-contained-html --junit-xml=report2-junit.xml --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout \|\| exit_code2=$?
		- zero_failures=$(cat report-junit.xml report2-junit.xml \| grep -c 'failures="0"') \|\| true

		- if [ $zero_failures != 1 ] && [ $non_interop_flag == 0 ]; then echo "Non-interop cases without non-interop flag encountered!"; exit $EXIT_CODE_FAIL; fi
		- if [ $zero_failures != 1 ] && [ $non_interop_flag == 1 ]; then echo "Non-interop cases with non-interop flag encountered"; exit $EXIT_CODE_NON_BE; fi
		- if [ $zero_failures != 2 ] && [ $non_interop_flag == 0 ]; then echo "Non-interop cases without non-interop flag encountered!"; exit $EXIT_CODE_FAIL; fi
		- if [ $zero_failures != 2 ] && [ $non_interop_flag == 1 ]; then echo "Non-interop cases with non-interop flag encountered"; exit $EXIT_CODE_NON_BE; fi
		- exit 0

		allow_failure:
		@@ -995,6 +1002,8 @@ ivas-interop-on-merge-request:
		paths:
		- report-junit.xml
		- report.html
		- report2-junit.xml
		- report2.html
		expose_as: "interop test results"
		reports:
		junit:
		@@ -1025,7 +1034,7 @@ evs-pytest-on-merge-request:
		### prepare pytest
		# create references
		- testcase_timeout=60
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 --param_file scripts/config/self_test_evs.prm --testcase_timeout=$testcase_timeout
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm --testcase_timeout=$testcase_timeout

		### Run test using branch scripts and input
		- if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi
		@@ -1309,7 +1318,8 @@ codec-comparison-on-main-push:
		- mv IVAS_dec_test IVAS_dec
		# create references
		- testcase_timeout=60
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 --testcase_timeout=$testcase_timeout
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref --testcase_timeout=$testcase_timeout
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref_part2 --testcase_timeout=$testcase_timeout

		### re-checkout the latest commit here, if ref_using_main is set
		- if [ $ref_using_main -eq 1 ]; then git checkout $latest_commit;fi
		@@ -1388,7 +1398,8 @@ ivas-conformance:
		# Reference creation
		- python scripts/prepare_combined_format_inputs.py
		- $TEST_SET = "tests/codec_be_on_mr_nonselection", "tests/renderer/test_renderer.py", "tests/split_rendering/test_split_rendering.py"
		- python -m pytest $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files
		- python -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref --create_ref --keep_files
		- python -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref_part2 --keep_files

		# Output creation
		- python -m pytest $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html
		@@ -1480,7 +1491,8 @@ ivas-conformance-linux:
		# Reference creation
		- python3 scripts/prepare_combined_format_inputs.py
		- TEST_SET="tests/codec_be_on_mr_nonselection tests/renderer/test_renderer.py tests/split_rendering/test_split_rendering.py"
		- python3 -m pytest $TEST_SET -v -n auto --update_ref 1 --create_ref --keep_files
		- python3 -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref --create_ref --keep_files
		- python3 -m pytest $TEST_SET -v -n auto --update_ref 1 -m create_ref_part2 --keep_files

		# Output creation
		- python3 -m pytest $TEST_SET -v -n auto --keep_files --create_cut --html=report_cmd.html --self-contained-html
		@@ -1612,7 +1624,7 @@ test-long-self-test:
		# create references
		- exit_code_ref=0
		- testcase_timeout=300
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 --param_file scripts/config/self_test_ltv.prm --testcase_timeout=$testcase_timeout \|\| exit_code_ref=$?
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v --update_ref 1 -m create_ref --param_file scripts/config/self_test_ltv.prm --testcase_timeout=$testcase_timeout \|\| exit_code_ref=$?

		### run pytest self-test using long test vectors
		- exit_code=0
		@@ -2235,27 +2247,24 @@ coverage-test-on-main-scheduled:
		- *copy-ltv-files-to-testv-dir
		- make GCOV=1 -j
		- cp IVAS_rend IVAS_rend_ref # Copy exec to be able to run renderer script

		# -- Run short test vector suite to check coverage of that first
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec

		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv
		# need to ignore non-zero exit codes as limiter is active and thus the different framesiszes will not be BE in all cases
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only \|\| true
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only \|\| true
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv \|\| true
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv \|\| true
		- python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_ref
		- python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_cut
		- lcov -c -d obj -o coverage_stv.info # extract coverage of short test vectors here
		# remove apps and lib_util files from coverage
		- lcov -r coverage_stv.info "apps" -o coverage_stv.info
		- lcov -r coverage_stv.info "lib_util" -o coverage_stv.info

		# -- Add extended coverage of EVS test and long test vectors
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec
		- bash ci/smoke_test.sh coverage
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v -n auto --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec --use_ltv
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only --use_ltv \|\| true
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_sba.py -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only --use_ltv \|\| true
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR/test_param_file.py -v -n auto --update_ref 1 -m create_ref --param_file scripts/config/self_test_ltv.prm --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec
		- python3 -m pytest tests/test_be_for_jbm_neutral_dly_profile.py
		- lcov -c -d obj -o coverage.info
		# remove apps and lib_util files from coverage