Merge branch 'ci/add-conformance-basop' into 'main' (4b1f80b0) · Commits · IVAS Codec Public Collaboration / IVAS Codec CI

main-basop.yml

+51 −3

Original line number	Diff line number	Diff line
		@@ -80,6 +80,9 @@ workflow:
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test'
		variables:
		IVAS_PIPELINE_NAME: 'Test long self-test against main pipeline: $CI_COMMIT_BRANCH'
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux'
		variables:
		IVAS_PIPELINE_NAME: 'IVAS Conformance Linux: $CI_COMMIT_BRANCH'
		- if: $CI_PIPELINE_SOURCE == 'schedule' # Scheduled in any branch
		variables:
		IVAS_PIPELINE_NAME: 'Scheduled pipeline: $CI_COMMIT_BRANCH'
		@@ -149,6 +152,8 @@ workflow:

		.rules-pytest-to-ref-pre:
		rules:
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == "ivas-conformance-linux"
		when: never
		- if: $CI_PIPELINE_SOURCE == 'web'
		- if: $CI_PIPELINE_SOURCE == 'schedule'
		- if: $CI_PIPELINE_SOURCE == 'push'
		@@ -950,7 +955,7 @@ pytest-to-ref-pre:
		# ensure that codec builds on linux
		build-codec-linux-make:
		rules:
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $UPDATE_PAGES
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $UPDATE_PAGES
		when: never
		- if: $CI_PIPELINE_SOURCE == 'web'
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
		@@ -970,7 +975,7 @@ build-codec-linux-cmake:
		rules:
		- if: $CI_MERGE_REQUEST_TITLE =~ /^(\[Draft\]\|$Draft$\|Draft:)/
		when: never
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $UPDATE_PAGES
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' \|\| $UPDATE_PAGES
		when: never
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
		@@ -1055,7 +1060,7 @@ build-codec-linux-debugging-make:

		build-codec-windows-msbuild:
		rules:
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $UPDATE_PAGES
		- if: $MANUAL_PIPELINE_TYPE == 'long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'backup-long-term-logs' \|\| $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux' \|\| $UPDATE_PAGES
		when: never
		- if: $CI_PIPELINE_SOURCE == 'web'
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
		@@ -3171,6 +3176,49 @@ complexity-osba-in-binaural_room_ir-out:
		- *complexity-measurements-report-summary
		- exit $ret_val

		ivas-conformance-linux:
		extends:
		- .job-linux
		stage: test
		timeout: "6 hours"
		rules:
		- if: ($CI_PIPELINE_SOURCE == 'web' \|\| $CI_PIPELINE_SOURCE == 'trigger') && $MANUAL_PIPELINE_TYPE == 'ivas-conformance-linux'
		- if: $MANUAL_PIPELINE_TYPE == "coverage"
		- if: $COVERAGE_TEST
		allow_failure:
		exit_codes:
		- 123
		variables:
		COVERAGE_OUTPUT_FILE: "coverage_conformance.info"
		COVERAGE_OUTPUT_DIR: "coverage_conformance"
		COVERAGE_TITLE: "conformance test test_26252.py"
		script:
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/print-common-info.sh
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/basop/update-scripts-repo.sh
		- python3 ci/remove_unsupported_testcases.py $PRM_FILES # Should not be needed once the porting is done and all test cases are supported in BASOP/float.
		- make -j # To be substituted with reference platform build
		- bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/ivas-conformance.sh

		artifacts:
		name: "ivas-conformance-linux-$CI_COMMIT_SHORT_SHA"
		expire_in: 1 week
		when: always
		paths:
		- report_cmd.html
		- report-junit.xml
		- report.html
		- Readme_IVAS_dec.txt
		- Readme_IVAS_enc.txt
		- Readme_IVAS_rend.txt
		- Readme_IVAS_JBM_dec.txt
		- Readme_IVAS_ISAR_dec.txt
		- Readme_IVAS_ISAR_post_rend.txt
		- $COVERAGE_OUTPUT_FILE
		- $COVERAGE_OUTPUT_DIR
		expose_as: "Draft IVAS conformance -- Linux"
		reports:
		junit: report-junit.xml

		# job that sets up gitlab pages website
		pages:
		stage: deploy

snippets/ivas-conformance.sh

0 → 100644

+68 −0

Original line number	Diff line number	Diff line
		#! /bin/bash

		set -euxo pipefail

		cp IVAS_cod IVAS_cod_ref
		cp IVAS_dec IVAS_dec_ref
		cp IVAS_rend IVAS_rend_ref
		cp ISAR_post_rend ISAR_post_rend_ref

		# Reference creation
		python3 -m pytest -q tests/codec_be_on_mr_nonselection tests/renderer_short/test_renderer.py tests/split_rendering/test_split_rendering.py -v -n auto --update_ref 1 --create_ref --keep_files --html=report_cmd.html --self-contained-html
		python3 scripts/parse_commands.py report_cmd.html Readme_IVAS.txt

		# Copy input data and output ref data
		rm -rf testvec
		mkdir testvec
		mkdir testvec/binauralRenderer_interface
		mkdir testvec/testv
		mkdir testvec/testv/renderer_short
		mkdir testvec/testv/split_rendering
		mkdir testvec/bin
		cp -r scripts/testv/* testvec/testv
		cp -r scripts/ls_layouts testvec
		cp -r scripts/object_edit testvec
		cp -r scripts/switchPaths testvec
		cp -r scripts/trajectories testvec
		cp -r scripts/binauralRenderer_interface/binaural_renderers_hrtf_data testvec/binauralRenderer_interface
		cp -r tests/ref testvec/testv/ref
		cp -r tests/renderer_short/ref testvec/testv/renderer_short/ref
		cp -r tests/split_rendering/ref testvec/testv/split_rendering/ref
		cp -r tests/split_rendering/renderer_configs testvec/testv/split_rendering/renderer_configs
		cp -r tests/split_rendering/error_patterns testvec/testv/split_rendering/error_patterns

		# Remove redundant files
		python3 scripts/cleanup_26252.py

		# Copy test script files
		cp -r tests/conformance-test testvec/
		cp Readme_IVAS_dec.txt testvec
		cp Readme_IVAS_enc.txt testvec
		cp Readme_IVAS_rend.txt testvec
		cp Readme_IVAS_JBM_dec.txt testvec
		cp Readme_IVAS_ISAR_dec.txt testvec
		cp Readme_IVAS_ISAR_post_rend.txt testvec

		# Create GCOV execs for coverage analysis
		make clean
		make GCOV=1 -j

		cp IVAS_cod testvec/bin
		cp IVAS_dec testvec/bin
		cp IVAS_rend testvec/bin
		cp ISAR_post_rend testvec/bin

		# Test run generated scripts in testvec
		cd testvec
		exit_code=0
		python3 -m pytest -q conformance-test/test_26252.py --junit-xml=report-junit.xml --html=report.html --self-contained-html \|\| exit_code=$?
		mv report.html ..
		mv report-junit.xml ..

		# Collect coverage
		cd -
		bash "${CI_PROJECT_DIR}"/ivas-codec-ci/snippets/collect-coverage.sh --obj-dir obj --output-file $COVERAGE_OUTPUT_FILE --output-dir $COVERAGE_OUTPUT_DIR --title "Coverage on main -- $COVERAGE_TITLE"

		# Check for failures
		if [ $exit_code -ne 0 ]; then echo "Test failures encountered"; exit $EXIT_CODE_FAIL; fi