Merge branch 'main' of ssh://forge.3gpp.org:29419/ivas-codec-pc/ivas-codec-ci... (357012cf) · Commits · IVAS Codec Public Collaboration / IVAS Codec CI

.gitlab-ci.yml

0 → 100644

+139 −0

Original line number	Diff line number	Diff line
		default:
		interruptible: true

		stages:
		- test

		workflow:
		rules:
		- if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push"
		when: never
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event'

		# Get the merged CI config for ivas-codec and ivas-basop repos based on the current branch and check for it to be valid.
		check-merged-ci-config-is-valid:
		stage: test
		tags:
		- ivas-linux

		variables:
		REPO_URL_FLOAT: "https://forge.3gpp.org/rep/ivas-codec-pc/ivas-codec"
		REPO_URL_BASOP: "https://forge.3gpp.org/rep/sa4/audio/ivas-basop"
		parallel:
		matrix:
		- REPO:
		- ivas-codec
		- ivas-basop

		script:
		- set -euo pipefail

		- \|
		if [ "$REPO" == "ivas-codec" ]; then
		repo_url=$REPO_URL_FLOAT
		patch_yml="main-float.yml"
		else
		repo_url=$REPO_URL_BASOP
		patch_yml="main-basop.yml"
		fi

		# artifact names
		- MERGED_RESPONSE="${REPO}-merged.json"
		- BASELINE_RESPONSE="${REPO}-baseline.json"
		- ARTIFACTS_MERGED_YAML="${REPO}-merged.yml"
		- ARTIFACTS_BASELINE_YAML="${REPO}-baseline.yml"
		- ARTIFACTS_DIFF_YAML="${REPO}-diff.yml"
		- TEST_YML="${REPO}-ci.yml"
		- BASE_YML="${REPO}-base-ci.yml"

		# Approach for validating the CI configs:
		# Due to intricacies of gitlab API token perissions/setup, the setup is a bit convoluted. We can't
		# just call the the lint API endpoint of the repective repos, but instead need to
		# - get the CI file from the respective repo (which includes from this repo's main)
		# - patch the include section so that it includes the correct local file in this repo (main-float.yml or main-basop-yml)
		# - call CI lint in THIS repo with passing the patched file -> this should have the same result as the other repo included remotely
		# - to get the merged yaml back, we need to pass dry_run: true, BUT this causes CI lint to actually run a pipeline which fails
		# because there are no push pipeline defined...
		# - so now we need to use the merged yaml file that we got back to again run the static lint and then finally we get a result for the whole thing being valid

		# Get CI file from the respective repo
		- curl -sS --url "${repo_url}/-/raw/main/.gitlab-ci.yml?inline=false" --output $TEST_YML

		- cp $TEST_YML $BASE_YML
		- sed -i '/^include:/,$d' $BASE_YML
		- \|
		echo -e "include:\n - local: ${patch_yml}" >> $BASE_YML

		# Get merged baseline with target branch ref
		- \|
		jq -Rs '{ dry_run: true, content: . }' < $BASE_YML \
		\| curl -sS -X POST \
		--url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true&dry_run=true&ref=${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}" \
		-H "Content-Type: application/json" \
		-H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \
		--output $BASELINE_RESPONSE \
		--data-binary @-

		# lint the whole merged thing statically (previous call tries to run pipeline which does not work because ivas-codec does not have push pipelines...)
		- jq -r ".merged_yaml" $BASELINE_RESPONSE > $ARTIFACTS_BASELINE_YAML
		- \|
		jq -Rs '{ content: . }' < $ARTIFACTS_BASELINE_YAML \
		\| curl -sS -X POST \
		--url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?" \
		-H "Content-Type: application/json" \
		-H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \
		--output $BASELINE_RESPONSE \
		--data-binary @-

		- sed -i '/^include:/,$d' $TEST_YML
		- \|
		echo -e "include:\n - local: ${patch_yml}" >> $TEST_YML

		# Get current with feature branch ref
		- \|
		jq -Rs '{ dry_run: true, content: . }' < $TEST_YML \
		\| curl -sS -X POST \
		--url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?include_merged_yaml=true&dry_run=true&ref=${CI_COMMIT_REF_NAME}" \
		-H "Content-Type: application/json" \
		-H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \
		--output $MERGED_RESPONSE \
		--data-binary @-

		# lint the whole merged thing statically (previous call tries to run pipeline which does not work because ivas-codec does not have push pipelines...)
		- jq -r ".merged_yaml" $MERGED_RESPONSE > $ARTIFACTS_MERGED_YAML
		- \|
		jq -Rs '{ content: . }' <$ARTIFACTS_MERGED_YAML \
		\| curl -sS -X POST \
		--url "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/ci/lint?" \
		-H "Content-Type: application/json" \
		-H "PRIVATE-TOKEN: $CI_LINT_TOKEN" \
		--output $MERGED_RESPONSE \
		--data-binary @-

		# find semantic diff between merged yaml with main and with this branch
		- dyff between $ARTIFACTS_BASELINE_YAML $ARTIFACTS_MERGED_YAML \| tee $ARTIFACTS_DIFF_YAML \|\| true

		# Check validity
		- baseline_valid=$(jq -r ".valid" $BASELINE_RESPONSE)
		- merged_valid=$(jq -r ".valid" $MERGED_RESPONSE)
		- \|
		if [ "$baseline_valid" != "true" ]; then
		echo "=== WARNING: BASELINE CONFIG IS INVALID ==="
		jq -r ".errors" $BASELINE_RESPONSE
		fi
		- \|
		if [ "$merged_valid" != "true" ]; then
		echo "=== MERGED CONFIG IS INVALID ==="
		jq -r ".errors" $MERGED_RESPONSE
		exit 1
		fi
		artifacts:
		when: always
		access: all
		expire_in: "5 days"
		paths:
		- "*-baseline.yml"
		- "*-merged.yml"
		- "*-diff.yml"
		- "*-baseline.json"
		- "*-merged.json"

main-basop.yml

+3 −3

Original line number	Diff line number	Diff line
		@@ -2201,7 +2201,7 @@ coverage-test-on-main-scheduled:

		- testcase_timeout=$TESTCASE_TIMEOUT_LTV
		- exit_code=0
		- python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection tests/renderer --update_ref 1 -v --create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml -n auto --testcase_timeout $testcase_timeout --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec \|\| exit_code=$?
		- python3 -m pytest --tb=no tests/codec_be_on_mr_nonselection tests/renderer_short --update_ref 1 -v --create_ref --html=report.html --self-contained-html --junit-xml=report-junit.xml -n auto --testcase_timeout $testcase_timeout --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec \|\| exit_code=$?
		- lcov -c -d obj -o coverage_enc_dec_rend.info # extract coverage

		# remove apps and lib_util files from coverage
		@@ -2486,8 +2486,8 @@ ivas-long-term-regressions:
		- for MEASURE in MLD MAX_ABS_DIFF; do
		- python3 scripts/find_regressions_from_logs.py logs regressions_"$MEASURE".csv --measure "$MEASURE" --days 7 --ratio_thr 1.1
		- mkdir regressions_"$MEASURE"
		- python3 scripts/generate_scripts_from_regressions.py regressions_"$MEASURE".csv
		- mv regression*bash regressions_"$MEASURE"
		- python3 scripts/generate_scripts_from_regressions.py regressions_"$MEASURE".csv # Outputs regressions*bash scripts if regressions are found
		- mv regressionbash regressions_"$MEASURE" \|\| true # Move regressionsbash scripts if they exists. If none exist, use \|\| to prevent failure
		- done

main-float.yml

+11 −7

Original line number	Diff line number	Diff line
		@@ -317,7 +317,7 @@ branch-is-up-to-date-with-main-post:
		# BASOP renderer test uses cut input signals
		- \|
		if [ "$TEST_SUITE" == "tests/renderer_short/test_renderer.py" ]; then
		python3 tests/create_short_testvectors --inplace --all --cut_len 5.0
		python3 tests/create_short_testvectors.py --inplace --all --cut_len 5.0
		fi
		- python3 -m pytest $TEST_SUITE $PYTEST_ARGS --update_ref 1 --create_ref --ref_encoder_path IVAS_cod --ref_decoder_path IVAS_dec --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout

		@@ -337,6 +337,7 @@ branch-is-up-to-date-with-main-post:
		basop-compat-stv:
		extends:
		- .basop-ci-branch-compat-template
		timeout: 120 minutes
		variables:
		TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_STV_SANITIZERS
		parallel:
		@@ -351,6 +352,7 @@ basop-compat-stv:
		basop-compat-encoder-dmx-comp:
		extends:
		- .basop-ci-branch-compat-template
		timeout: 120 minutes
		variables:
		TEST_SUITE: $SHORT_TEST_SUITE_ENCODER
		# USING PYTEST_ADDOPTS env var did not work for some reason when testing locally - maybe because this is a custom option
		@@ -362,6 +364,7 @@ basop-compat-encoder-dmx-comp:
		basop-compat-ltv:
		extends:
		- .basop-ci-branch-compat-template
		timeout: 600 minutes
		variables:
		TEST_SUITE: $LONG_TEST_SUITE
		TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_LTV_SANITIZERS
		@@ -371,6 +374,7 @@ basop-compat-ltv:
		basop-compat-encoder-ltv:
		extends:
		- .basop-ci-branch-compat-template
		timeout: 600 minutes
		variables:
		TEST_SUITE: $LONG_TEST_SUITE_ENCODER
		TESTCASE_TIMEOUT: $TESTCASE_TIMEOUT_LTV_SANITIZERS
		@@ -406,7 +410,7 @@ basop-compat-smoke-test:
		when: manual
		- if: $CI_PIPELINE_SOURCE == 'push'
		when: never
		timeout: "20 minutes"
		timeout: "120 minutes"
		stage: basop-compat
		needs: []
		allow_failure: true
		@@ -628,7 +632,7 @@ renderer-pytest-on-merge-request:
		- if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi

		# run test
		- python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout \|\| exit_code=$?
		- python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/renderer/test_renderer.py --create_cut --testcase_timeout=$testcase_timeout \|\| exit_code=$?
		- zero_errors=$(cat report-junit.xml \| grep -c 'errors="0"') \|\| true

		- *merge-request-comparison-check
		@@ -779,7 +783,7 @@ split-rendering-pytest-on-merge-request:
		- if [ $ref_using_main == 1 ]; then git checkout $source_branch_commit_sha; fi

		# run test
		- python3 -m pytest -q --log-level ERROR -n auto -rA --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_cut --testcase_timeout=$testcase_timeout \|\| exit_code=$?
		- python3 -m pytest -q --log-level ERROR -n auto -rA --mld --ssnr --html=report.html --self-contained-html --junit-xml=report-junit.xml tests/split_rendering/test_split_rendering.py --create_cut --testcase_timeout=$testcase_timeout \|\| exit_code=$?
		- zero_errors=$(cat report-junit.xml \| grep -c 'errors="0"') \|\| true

		- *merge-request-comparison-check
		@@ -838,7 +842,7 @@ ivas-pytest-on-merge-request:
		### run pytest
		- exit_code=0
		- testcase_timeout=60
		- python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT" -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout \|\| exit_code=$?
		- python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT" -v --mld --ssnr --html=report.html --self-contained-html --junit-xml=report-junit.xml --testcase_timeout=$testcase_timeout \|\| exit_code=$?
		- zero_errors=$(cat report-junit.xml \| grep -c 'errors="0"') \|\| true

		- *merge-request-comparison-check
		@@ -886,7 +890,7 @@ ivas-interop-on-merge-request:
		- exit_code2=0
		# set timeout for individual testcase runs to 60 seconds
		- testcase_timeout=60
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout \|\| exit_code=$?
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --mld --ssnr --html=report.html --self-contained-html --junit-xml=report-junit.xml --update_ref 1 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec_ref --testcase_timeout=$testcase_timeout \|\| exit_code=$?
		- zero_failures=$(cat report-junit.xml \| grep -c 'failures="0"') \|\| true

		- if [ $zero_failures != 1 ] && [ $non_interop_flag == 0 ]; then echo "Non-interop cases without non-interop flag encountered!"; exit $EXIT_CODE_FAIL; fi
		@@ -1522,7 +1526,7 @@ test-long-self-test:

		### run pytest self-test using long test vectors
		- exit_code=0
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --use_ltv -v --param_file scripts/config/self_test_ltv.prm --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml --testcase_timeout=$testcase_timeout \|\| exit_code=$?
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR --use_ltv -v --mld --ssnr --param_file scripts/config/self_test_ltv.prm --html=report-ltv.html --self-contained-html --junit-xml=report-junit-ltv.xml --testcase_timeout=$testcase_timeout \|\| exit_code=$?
		- zero_errors=$(cat report-junit-ltv.xml \| grep -c 'errors="0"') \|\| true

		- if [ $zero_errors != 1 ]; then echo "Run errors encountered!"; exit $EXIT_CODE_FAIL; fi