fix check-be tests (ff8a621a) · Commits · SA4 / Audio / IVAS BASOP

.gitlab-ci.yml

+56 −25

Original line number	Diff line number	Diff line
		@@ -2,7 +2,7 @@ variables:
		TESTV_DIR: "/usr/local/testv"
		LTV_DIR: "/usr/local/ltv"
		EVS_BE_TEST_DIR_BASOP: "/usr/local/be_2_evs_basop"
		REFERENCE_BRANCH: "ivas-float-update"
		FLOAT_REF_BRANCH: "ivas-float-update"
		BUILD_OUTPUT: "build_output.txt"
		SCRIPTS_DIR: "/usr/local/scripts"
		EXIT_CODE_NON_BE: 123
		@@ -17,6 +17,8 @@ variables:
		DUT_DECODER_PATH: "./IVAS_dec"
		REF_ENCODER_PATH: "./IVAS_cod_ref"
		REF_DECODER_PATH: "./IVAS_dec_ref"
		MERGE_TARGET_ENCODER_PATH: "./IVAS_cod_merge_target"
		MERGE_TARGET_DECODER_PATH: "./IVAS_dec_merge_target"
		LEVEL_SCALING: "1.0"
		IVAS_PIPELINE_NAME: ''
		BASOP_CI_BRANCH_PC_REPO: "basop-ci-branch"
		@@ -37,8 +39,9 @@ variables:
		FAILED_TESTCASES_LIST: "failed-testcases.txt"
		ERRORS_TESTCASES_LIST: "errors-testcases.txt"
		PYTEST_CACHE_ARTIFACT: "pytest_cache.zip"
		REF_COMMIT_FILE: "ref-branch-git-sha"
		CUT_COMMIT_FILE: "CuT-branch-git-sha"
		FLOAT_REF_COMMIT_FILE: "float-ref-git-sha"
		CUT_COMMIT_FILE: "CuT-git-sha"
		MERGE_TARGET_COMMIT_FILE: "merge-target-git-sha"
		MANUAL_PIPELINE_TYPE:
		description: "Type for the manual pipeline run. Use 'pytest-compare' to run comparison test against reference float codec."
		value: 'default'
		@@ -147,14 +150,13 @@ stages:
		- sed -i.bak -e "s/\/\\ $#define\ DEBUG_MODE_INFO\ $\*\//\1/g" lib_com/options.h
		- fi

		.build-reference-binaries: &build-reference-binaries
		.build-float-ref-binaries: &build-float-ref-binaries
		- git rev-parse HEAD > $CUT_COMMIT_FILE
		- current_commit_sha=$(git rev-parse HEAD)
		### build reference binaries
		- git checkout $REFERENCE_BRANCH
		- git checkout $FLOAT_REF_BRANCH
		- git pull
		- *activate-debug-mode-info-if-set
		- cat lib_com/options.h
		- make clean
		- make -j
		- mv ./IVAS_cod ./$REF_ENCODER_PATH
		@@ -162,20 +164,35 @@ stages:
		- mv ./IVAS_rend ./IVAS_rend_ref
		### Return to current branch
		- git restore .
		- git rev-parse HEAD > $REF_COMMIT_FILE
		- git rev-parse HEAD > $FLOAT_REF_COMMIT_FILE
		- git checkout $current_commit_sha

		.build-merge-target-binaries: &build-merge-target-binaries
		- current_commit_sha=$(git rev-parse HEAD)
		### build merge target binaries
		- git checkout $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
		- git pull
		- *activate-debug-mode-info-if-set
		- make clean
		- make -j
		- mv ./IVAS_cod ./$MERGE_TARGET_ENCODER_PATH
		- mv ./IVAS_dec ./$MERGE_TARGET_DECODER_PATH
		- mv ./IVAS_rend ./IVAS_rend_merge_target
		### Return to current branch
		- git restore .
		- git rev-parse HEAD > $MERGE_TARGET_COMMIT_FILE
		- git checkout $current_commit_sha

		.build-reference-and-dut-binaries: &build-reference-and-dut-binaries
		.build-float-ref-and-dut-binaries: &build-float-ref-and-dut-binaries
		### build reference binaries
		- *build-reference-binaries
		- *build-float-ref-binaries
		### build dut binaries
		- *activate-debug-mode-info-if-set
		- make clean
		- make -j

		.build-and-create-reference-outputs: &build-and-create-reference-outputs
		- *build-reference-and-dut-binaries
		.build-and-create-float-ref-outputs: &build-and-create-float-ref-outputs
		- *build-float-ref-and-dut-binaries

		### prepare pytest
		# create short test vectors
		@@ -365,7 +382,7 @@ stages:
		- if [ "$COMPARE_DMX" = "true" ] \|\| [ "$ENCODER_TEST" = "true" ]; then
		- BUILD_WITH_DEBUG_MODE_INFO="true"
		- fi
		- *build-and-create-reference-outputs
		- *build-and-create-float-ref-outputs

		- comp_args="--mld --ssnr --odg"
		- if [ "$ENCODER_TEST" = "true" ]; then
		@@ -448,15 +465,16 @@ stages:
		needs: ["build-codec-linux-make"]
		timeout: "300 minutes"
		variables:
		REFERENCE_BRANCH: "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME"
		XML_REPORT: "report--$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.xml"
		HTML_REPORT: "report--$CI_JOB_NAME--sha-$CI_COMMIT_SHORT_SHA.html"
		PYTEST_LOG_TARGET_BRANCH: "pytest-log-$CI_MERGE_REQUEST_TARGET_BRANCH_NAME.txt"
		script:
		- *print-common-info

		- set -euxo pipefail

		- *update-scripts-repo
		- python3 tests/create_short_testvectors.py

		- if [ $USE_LTV -eq 1 ]; then
		- *update-ltv-repo
		@@ -471,12 +489,16 @@ stages:
		- *apply-testv-scaling
		- fi

		- *build-and-create-reference-outputs
		- *build-float-ref-binaries
		- *build-merge-target-binaries
		- *check-up-to-date-in-comparison-jobs

		- exit_code_target=0
		- python3 -m pytest $TEST_SUITE -v --update_ref 1 --create_ref -n auto --ref_encoder_path $MERGE_TARGET_ENCODER_PATH --ref_decoder_path $MERGE_TARGET_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH > $PYTEST_LOG_TARGET_BRANCH \|\| exit_code_target=$?

		- exit_code=0
		- rm -rf .pytest_cache \|\| true
		- python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --mld --ssnr --odg --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt \|\| exit_code=$?
		- python3 -m pytest --tb=no -q $TEST_SUITE -v --keep_files --create_cut --html=$HTML_REPORT --self-contained-html --junit-xml=$XML_REPORT --mld --ssnr --odg --ref_encoder_path $MERGE_TARGET_ENCODER_PATH --ref_decoder_path $MERGE_TARGET_DECODER_PATH --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH -n auto --testcase_timeout $testcase_timeout > pytest_log.txt \|\| exit_code=$?

		- if [ $exit_code -ne 0 ]; then
		- exit_code=$EXIT_CODE_NON_BE
		@@ -495,7 +517,7 @@ stages:
		- cat failed_testcases_for_printing.txt

		- echo "Reproduce locally with:"
		- echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH"
		- echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $FLOAT_REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH"
		- echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH"
		- echo "The individual command lines can be found in the html report in the job artifacts."
		- if [ $num_errors -ne 0 ]; then
		@@ -529,8 +551,10 @@ stages:
		- $ERRORS_TESTCASES_LIST
		- pytest_log.txt
		- $PYTEST_CACHE_ARTIFACT
		- $REF_COMMIT_FILE
		- $FLOAT_REF_COMMIT_FILE
		- $CUT_COMMIT_FILE
		- $MERGE_TARGET_COMMIT_FILE
		- $PYTEST_LOG_TARGET_BRANCH
		expose_as: "pytest compare results"
		reports:
		junit:
		@@ -591,7 +615,7 @@ stages:

		### run branch first
		# this per default builds the branch and the reference and creates the reference outputs
		- *build-and-create-reference-outputs
		- *build-and-create-float-ref-outputs
		- *check-up-to-date-in-comparison-jobs
		# need to restore cache again
		- *overwrite-pytest-cache-with-artifact
		@@ -627,7 +651,7 @@ stages:
		- echo "Run errors encountered!"
		- exit_code=$EXIT_CODE_FAIL
		- echo "Reproduce locally with:"
		- echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $ERRORS_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH"
		- echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $FLOAT_REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $ERRORS_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH"
		- echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $ERRORS_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH"
		- echo "The individual command lines can be found in the changes*.csv files in the job artifacts."
		- elif [ $regressions_found != 0 ] && [ "$SKIP_REGRESSION_CHECK" != "true" ]; then
		@@ -640,7 +664,7 @@ stages:
		- exit_code=$EXIT_CODE_NON_BE;
		- fi
		- echo "Reproduce locally with:"
		- echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH"
		- echo -e "1. Create references with target branch $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:\n\t- git checkout $(cat $FLOAT_REF_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- mv IVAS_cod IVAS_cod_ref\n\t- mv IVAS_dec IVAS_dec_ref\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --update_ref 1 --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path $REF_DECODER_PATH"
		- echo -e "2. Run test with source branch $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:\n\t- git checkout $(cat $CUT_COMMIT_FILE)\n\t- make clean\n\t- make -j\n\t- python3 -m pytest $(cat $FAILED_TESTCASES_LIST) --dut_encoder_path $DUT_ENCODER_PATH --dut_decoder_path $DUT_DECODER_PATH"
		- echo "The individual command lines can be found in the changes*.csv files in the job artifacts."
		- fi
		@@ -664,8 +688,9 @@ stages:
		- $CSV_MAIN
		- $SUMMARY_HTML_ARTIFACT_NAME
		- $IMAGES_ARTIFACT_NAME
		- $REF_COMMIT_FILE
		- $FLOAT_REF_COMMIT_FILE
		- $CUT_COMMIT_FILE
		- $MERGE_TARGET_COMMIT_FILE
		- changes_crashes.csv
		- changes_MLD.csv
		- changes_MAXIMUM_ABS_DIFF.csv
		@@ -695,7 +720,7 @@ stages:
		- *copy-ltv-files-to-testv-dir
		- python3 ci/remove_unsupported_testcases.py $PRM_FILES

		- *build-reference-binaries
		- *build-float-ref-binaries
		- set -euxo pipefail
		- make_args="CLANG=$CLANG_NUM"
		- if [[ $CLANG_NUM == 3 ]]; then
		@@ -740,7 +765,7 @@ stages:
		- if [ $LEVEL_SCALING != "1.0" ];then
		- *apply-testv-scaling
		- fi
		- *build-reference-and-dut-binaries
		- *build-float-ref-and-dut-binaries

		### run pytest
		- exit_code=0
		@@ -975,6 +1000,7 @@ check-be-to-target-short-enc-0db:
		before_script:
		- USE_LTV=0
		- DUT_DECODER_PATH=./IVAS_dec_ref
		- MERGE_TARGET_DECODER_PATH=./IVAS_dec_ref
		- TEST_SUITE="$SHORT_TEST_SUITE_ENCODER"
		- LEVEL_SCALING=1.0
		- rm -rf tests/dut tests/ref
		@@ -987,6 +1013,7 @@ check-be-to-target-short-enc-+10db:
		before_script:
		- USE_LTV=0
		- DUT_DECODER_PATH=./IVAS_dec_ref
		- MERGE_TARGET_DECODER_PATH=./IVAS_dec_ref
		- TEST_SUITE="$SHORT_TEST_SUITE_ENCODER"
		- LEVEL_SCALING=3.162
		- rm -rf tests/dut tests/ref
		@@ -999,6 +1026,7 @@ check-be-to-target-short-enc--10db:
		before_script:
		- USE_LTV=0
		- DUT_DECODER_PATH=./IVAS_dec_ref
		- MERGE_TARGET_DECODER_PATH=./IVAS_dec_ref
		- TEST_SUITE="$SHORT_TEST_SUITE_ENCODER"
		- LEVEL_SCALING=0.3162
		- rm -rf tests/dut tests/ref
		@@ -1011,6 +1039,7 @@ check-be-to-target-short-dec-0db:
		before_script:
		- USE_LTV=0
		- DUT_ENCODER_PATH=./IVAS_cod_ref
		- MERGE_TARGET_ENCODER_PATH=./IVAS_cod_ref
		- TEST_SUITE="$SHORT_TEST_SUITE"
		- LEVEL_SCALING=1.0
		- rm -rf tests/dut tests/ref
		@@ -1023,6 +1052,7 @@ check-be-to-target-short-dec-+10db:
		before_script:
		- USE_LTV=0
		- DUT_ENCODER_PATH=./IVAS_cod_ref
		- MERGE_TARGET_ENCODER_PATH=./IVAS_cod_ref
		- TEST_SUITE="$SHORT_TEST_SUITE"
		- LEVEL_SCALING=3.162
		- rm -rf tests/dut tests/ref
		@@ -1035,6 +1065,7 @@ check-be-to-target-short-dec--10db:
		before_script:
		- USE_LTV=0
		- DUT_ENCODER_PATH=./IVAS_cod_ref
		- MERGE_TARGET_ENCODER_PATH=./IVAS_cod_ref
		- TEST_SUITE="$SHORT_TEST_SUITE"
		- LEVEL_SCALING=0.3162
		- rm -rf tests/dut tests/ref
		@@ -1466,7 +1497,7 @@ coverage-test-on-main-scheduled:
		- *update-scripts-repo
		- *update-ltv-repo
		- *copy-ltv-files-to-testv-dir
		- *build-reference-binaries
		- *build-float-ref-binaries
		# Build DuT binaries with GCOV
		- make clean
		- make GCOV=1 -j
		@@ -1665,7 +1696,7 @@ voip-be-on-merge-request:
		- *print-common-info
		- *update-scripts-repo
		- *update-ltv-repo
		- *build-reference-and-dut-binaries
		- *build-float-ref-and-dut-binaries
		- *complexity-measurements-setup
		- which coan
		artifacts: