Merge branch 'main' into switch-basop-reference-to-ivas-codec-repo (ffe07fa4) · Commits · IVAS Codec Public Collaboration / IVAS Codec CI

main-basop.yml

+69 −18

Original line number	Diff line number	Diff line
		@@ -80,6 +80,9 @@ workflow:
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test'
		variables:
		IVAS_PIPELINE_NAME: 'Test long self-test against main pipeline: $CI_COMMIT_BRANCH'
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux'
		variables:
		IVAS_PIPELINE_NAME: 'IVAS Conformance Linux: $CI_COMMIT_BRANCH'
		- if: $CI_PIPELINE_SOURCE == 'schedule' # Scheduled in any branch
		variables:
		IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH'
		@@ -149,6 +152,8 @@ workflow:

		.rules-pytest-to-ref-pre:
		rules:
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "ivas-conformance-linux"
		when: never
		- if: $CI_PIPELINE_SOURCE == 'web'
		- if: $CI_PIPELINE_SOURCE == 'schedule'
		- if: $CI_PIPELINE_SOURCE == 'push'
		@@ -280,7 +285,6 @@ workflow:
		- FLOAT_REF_COMMIT=$(cat "$FLOAT_REF_COMMIT_FILE")
		- echo "FLOAT_REF_COMMIT=$FLOAT_REF_COMMIT"

		- python3 ci/remove_unsupported_testcases.py $PRM_FILES
		- if [ $LEVEL_SCALING != "1.0" ]; then
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh
		- fi
		@@ -289,6 +293,8 @@ workflow:
		- BUILD_WITH_DEBUG_MODE_INFO="true"
		- fi

		- export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not rtpdump'"

		- INV_LEVEL_SCALING=$(awk "BEGIN {print 1.0 / $LEVEL_SCALING}")
		- comp_args="--mld --ssnr --odg --scalefac $INV_LEVEL_SCALING"
		- summary_args="MLD DIFF SSNR ODG"
		@@ -428,6 +434,9 @@ workflow:
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh
		- fi

		# TODO: remove once ref branch is switched
		- export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not rtpdump'"

		# build merge target branch and correpsonding float reference
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref
		- source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-merge-target-binaries.sh
		@@ -435,8 +444,6 @@ workflow:
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh dut
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-binaries.sh float-ref-merge-source

		- python3 ci/remove_unsupported_testcases.py $PRM_FILES

		- exit_code_target=0
		- python3 -m pytest $TEST_SUITE -v --update_ref 1 --create_ref -n auto --ref_encoder_path $MERGE_TARGET_ENCODER_PATH --ref_decoder_path $MERGE_TARGET_DECODER_PATH > $PYTEST_LOG_TARGET_BRANCH \|\| exit_code_target=$?

		@@ -553,6 +560,9 @@ workflow:
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh
		- fi

		# TODO: remove once ref branch is switched
		- export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not rtpdump'"

		# check MR title for flag that allows regressions to be mergable
		- echo $CI_MERGE_REQUEST_TITLE > tmp.txt
		- allow_regressions_flag=$(grep -c --ignore-case "\[allow[ -]*regression\]" tmp.txt) \|\| true
		@@ -577,8 +587,6 @@ workflow:
		- enc_dmx_arg="--compare_enc_dmx"
		- fi

		- python3 ci/remove_unsupported_testcases.py $PRM_FILES

		### ----- run pytest for branch (dut) first -----
		# create float reference outputs with corresponding branch
		- python3 -m pytest $TEST_SUITE -v --update_ref 1 $enc_stats_arg $enc_dmx_arg --create_ref -n auto --ref_encoder_path $MERGE_SOURCE_FLOAT_REF_ENCODER_PATH --ref_decoder_path $MERGE_SOURCE_FLOAT_REF_DECODER_PATH \|\| exit_code=$?
		@@ -681,7 +689,6 @@ workflow:
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/copy-ltv-files-to-testv-dir.sh
		- python3 ci/remove_unsupported_testcases.py $PRM_FILES

		- set -euxo pipefail
		- make_args="CLANG=$CLANG_NUM"
		@@ -731,7 +738,6 @@ workflow:
		- testcase_timeout=$TESTCASE_TIMEOUT_STV
		- fi

		- python3 ci/remove_unsupported_testcases.py $PRM_FILES
		- if [ $LEVEL_SCALING != "1.0" ];then
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh
		- fi
		@@ -949,7 +955,7 @@ pytest-to-ref-pre:
		# ensure that codec builds on linux
		build-codec-linux-make:
		rules:
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $UPDATE_PAGES
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $UPDATE_PAGES
		when: never
		- if: $CI_PIPELINE_SOURCE == 'web'
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
		@@ -969,7 +975,7 @@ build-codec-linux-cmake:
		rules:
		- if: $CI_MERGE_REQUEST_TITLE =~ /^(\[Draft\]\|$Draft$\|Draft:)/
		when: never
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $UPDATE_PAGES
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' \|\| $UPDATE_PAGES
		when: never
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
		@@ -1054,7 +1060,7 @@ build-codec-linux-debugging-make:

		build-codec-windows-msbuild:
		rules:
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $UPDATE_PAGES
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' \|\| $UPDATE_PAGES
		when: never
		- if: $CI_PIPELINE_SOURCE == 'web'
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
		@@ -1337,7 +1343,6 @@ ivas-pytest-on-merge-request:
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh
		- source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/build-merge-target-binaries.sh
		- source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/merge-request-comparison-setup-codec.sh
		- python3 ci/remove_unsupported_testcases.py $PRM_FILES
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/merge-request-print-pinned-commits.sh

		# some helper variables - "\|\| true" to prevent failures from grep not finding anything
		@@ -1358,6 +1363,10 @@ ivas-pytest-on-merge-request:
		- export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not model'"
		- fi

		- if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "ivas-float-update" ]; then
		- export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not rtpdump'"
		- fi

		### prepare pytest
		# create references
		- python3 -m pytest "$TESTS_DIR_CODEC_BE_ON_MR"/"$PYTEST_SCRIPT" -v --update_ref 1
		@@ -1476,7 +1485,6 @@ codec-smoke-test:
		# LTV update needed as ltv ISM metadata files are used
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/update-ltv-repo.sh
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh
		- python3 scripts/smoketest-basop-filter.py scripts/config/ivas_modes.json --inline
		- bash ci/smoke_test.sh \|\| true
		- tar cJf logs.tar.xz out/logs/
		- ls -al logs.tar.*
		@@ -1567,7 +1575,9 @@ ivas-interop-on-merge-request:
		- source "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/merge-request-comparison-setup-codec.sh
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/merge-request-print-pinned-commits.sh

		- python3 ci/remove_unsupported_testcases.py $PRM_FILES
		- if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "ivas-float-update" ]; then
		- export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not rtpdump'"
		- fi

		# some helper variables - "\|\| true" to prevent failures from grep not finding anything
		# write to temporary file as workaround for failures observed with piping echo
		@@ -1788,7 +1798,7 @@ renderer-usan:
		- report-junit-5ms.xml

		# From float repo
		# to be reused in MR and LTV-scheduled sanitizer test jobs
		# to be reused in MR test jobs
		# set CLANG_NUM, SELFTEST_SANITY_TIMEOUT and SELF_TEST_PRM_FILE in before_script section
		.sanitizer-selftest-anchor: &sanitizer-selftest-anchor
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh
		@@ -1801,7 +1811,9 @@ renderer-usan:
		- export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not model'"
		- fi

		- python3 ci/remove_unsupported_testcases.py $PRM_FILES
		- if [ "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" == "ivas-float-update" ]; then
		- export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} -k 'not rtpdump'"
		- fi

		- make_args="CLANG=$CLANG_NUM"
		# for MRs to main, the whole sanitizer test takes too long (even with stv set of inputs)
		@@ -2242,7 +2254,6 @@ test-long-self-test:
		;;
		esac

		- python3 ci/remove_unsupported_testcases.py $PRM_FILES
		- if [ $LEVEL_SCALING != "1.0" ]; then
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/apply-testv-scaling.sh
		- fi
		@@ -2491,8 +2502,6 @@ check-be-between-renderer-framesizes:
		- make clean
		- make -j

		- python3 ci/remove_unsupported_testcases.py $PRM_FILES

		- exit_code=0
		# NOTE: here we do not use the "DUT_..CODER_PATH" variables because we do not build via a script, but directly from the make file (which is done because of the "make_args" - no way to inject that into build-binaries.sh)
		- python3 -m pytest tests/codec_be_on_mr_nonselection --tb=no -v --ref_encoder_path $REF_ENCODER_PATH --ref_decoder_path ./IVAS_dec --update_ref 1 --html=report-20ms.html --self-contained-html --junit-xml=report-junit-20ms.xml \|\| exit_code=$?
		@@ -3170,6 +3179,48 @@ complexity-osba-in-binaural_room_ir-out:
		- *complexity-measurements-report-summary
		- exit $ret_val

		ivas-conformance-linux:
		extends:
		- .job-linux
		stage: test
		timeout: "6 hours"
		rules:
		- if: ($CI_PIPELINE_SOURCE == 'web' \|\| $CI_PIPELINE_SOURCE == 'trigger') && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux'
		- if: $MANUAL_PIPELINE_TYPE == "coverage"
		- if: $COVERAGE_TEST
		allow_failure:
		exit_codes:
		- 123
		variables:
		COVERAGE_OUTPUT_FILE: "coverage_conformance.info"
		COVERAGE_OUTPUT_DIR: "coverage_conformance"
		COVERAGE_TITLE: "conformance test test_26252.py"
		script:
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh
		- make -j # To be substituted with reference platform build
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/ivas-conformance.sh

		artifacts:
		name: "ivas-conformance-linux-$CI_COMMIT_SHORT_SHA"
		expire_in: 1 week
		when: always
		paths:
		- report_cmd.html
		- report-junit.xml
		- report.html
		- Readme_IVAS_dec.txt
		- Readme_IVAS_enc.txt
		- Readme_IVAS_rend.txt
		- Readme_IVAS_JBM_dec.txt
		- Readme_IVAS_ISAR_dec.txt
		- Readme_IVAS_ISAR_post_rend.txt
		- $COVERAGE_OUTPUT_FILE
		- $COVERAGE_OUTPUT_DIR
		expose_as: "Draft IVAS conformance -- Linux"
		reports:
		junit: report-junit.xml

		# job that sets up gitlab pages website
		pages:
		stage: deploy

snippets/ivas-conformance.sh

0 → 100644

+68 −0

Original line number	Diff line number	Diff line
		#! /bin/bash

		set -euxo pipefail

		cp IVAS_cod IVAS_cod_ref
		cp IVAS_dec IVAS_dec_ref
		cp IVAS_rend IVAS_rend_ref
		cp ISAR_post_rend ISAR_post_rend_ref

		# Reference creation
		python3 -m pytest -q tests/codec_be_on_mr_nonselection tests/renderer_short/test_renderer.py tests/split_rendering/test_split_rendering.py -v -n auto --update_ref 1 --create_ref --keep_files --html=report_cmd.html --self-contained-html
		python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt

		# Copy input data and output ref data
		rm -rf testvec
		mkdir testvec
		mkdir testvec/binauralRenderer_interface
		mkdir testvec/testv
		mkdir testvec/testv/renderer_short
		mkdir testvec/testv/split_rendering
		mkdir testvec/bin
		cp -r scripts/testv/* testvec/testv
		cp -r scripts/ls_layouts testvec
		cp -r scripts/object_edit testvec
		cp -r scripts/switchPaths testvec
		cp -r scripts/trajectories testvec
		cp -r scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface
		cp -r tests/ref testvec/testv/ref
		cp -r tests/renderer_short/ref testvec/testv/renderer_short/ref
		cp -r tests/split_rendering/ref testvec/testv/split_rendering/ref
		cp -r tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs
		cp -r tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns

		# Remove redundant files
		python3 scripts/cleanup_26252.py

		# Copy test script files
		cp -r tests/conformance-test testvec/
		cp Readme_IVAS_dec.txt testvec
		cp Readme_IVAS_enc.txt testvec
		cp Readme_IVAS_rend.txt testvec
		cp Readme_IVAS_JBM_dec.txt testvec
		cp Readme_IVAS_ISAR_dec.txt testvec
		cp Readme_IVAS_ISAR_post_rend.txt testvec

		# Create GCOV execs for coverage analysis
		make clean
		make GCOV=1 -j

		cp IVAS_cod testvec/bin
		cp IVAS_dec testvec/bin
		cp IVAS_rend testvec/bin
		cp ISAR_post_rend testvec/bin

		# Test run generated scripts in testvec
		cd testvec
		exit_code=0
		python3 -m pytest -q conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html \|\| exit_code=$?
		mv report.html ..
		mv report-junit.xml ..

		# Collect coverage
		cd -
		bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/collect-coverage.sh --obj-dir obj --output-file $COVERAGE_OUTPUT_FILE --output-dir $COVERAGE_OUTPUT_DIR --title "Coverage on main -- $COVERAGE_TITLE"

		# Check for failures
		if [ $exit_code -ne 0 ]; then echo "Test failures encountered"; exit $EXIT_CODE_FAIL; fi