Merge branch 'ci/be-test-against-accepted-release-ci-job' into... (cff44693) · Commits · IVAS Codec Public Collaboration / IVAS Codec

.gitlab-ci.yml

+173 −17

Original line number	Diff line number	Diff line
		@@ -20,6 +20,12 @@ variables:
		- 'default'
		- 'test-be-release'
		- 'test-long-self-test'
		REF_COMMIT:
		description: "Ref commit id or ref release tag. To be used by Manual pipelines: 'test-be-release'"
		value: ''
		CUT_COMMIT:
		description: "Cut commit id or cut release tag. To be used by Manual pipeline:'test-be-release'"
		value: ''


		default:
		@@ -46,7 +52,7 @@ workflow:
		IVAS_PIPELINE_NAME: 'Web run pipeline: $CI_COMMIT_BRANCH'
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release'
		variables:
		IVAS_PIPELINE_NAME: 'Test BE to release pipeline: $CI_COMMIT_BRANCH'
		IVAS_PIPELINE_NAME: 'Test BE to release pipeline: $REF_COMMIT vs $CUT_COMMIT'
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-long-self-test'
		variables:
		IVAS_PIPELINE_NAME: 'Test long self-test against main pipeline: $CI_COMMIT_BRANCH'
		@@ -330,7 +336,7 @@ build-codec-instrumented-linux:
		extends:
		- .build-job-linux
		- .rules-basis
		timeout: "6 minutes"
		timeout: "7 minutes"
		script:
		- *print-common-info
		- bash ci/build_codec_instrumented_linux.sh
		@@ -1168,35 +1174,185 @@ codec-comparison-on-main-push:
		# Manual jobs
		# ---------------------------------------------------------------

		test-be-to-release:
		.print-common-info-ref-vs-cut: &print-common-info-ref-vs-cut
		- \|
		echo "Printing common information for build job."
		echo "REF_COMMIT: $REF_COMMIT."
		echo "CUT_COMMIT: $CUT_COMMIT."
		echo "Current job is run on commit $CI_COMMIT_SHA"
		echo "Commit time was $CI_COMMIT_TIMESTAMP"
		date \| xargs echo "System time is"

		.comparison-setup-codec-ref-vs-cut:
		&comparison-setup-codec-ref-vs-cut
		- git checkout $REF_COMMIT
		- $winoutdata = $null
		- MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /m \| tee -variable winoutdata
		- $winoutdata \| Out-File $BUILD_OUTPUT -Encoding Utf8
		- mv IVAS_cod.exe IVAS_cod_ref.exe
		- mv IVAS_dec.exe IVAS_dec_ref.exe
		- mv IVAS_rend.exe IVAS_rend_ref.exe
		- mkdir ref_hrtf_bin
		- cp scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/* ref_hrtf_bin

		- git checkout $CUT_COMMIT
		- MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /m \| tee -variable winoutdata
		- $winoutdata \| Out-File $BUILD_OUTPUT -Encoding Utf8
		- mv IVAS_cod.exe IVAS_cod.exe
		- mv IVAS_dec.exe IVAS_dec.exe
		- mv IVAS_rend.exe IVAS_rend.exe
		- mkdir cut_hrtf_bin
		- cp scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/* cut_hrtf_bin

		# switch to $CI_COMMIT_SHORT_SHA where the scripts reside
		- git checkout $CI_COMMIT_SHORT_SHA

		.rules-manual-pipeline:
		rules:
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE =~ 'default'

		.rules-test-be-release:
		extends: .rules-manual-pipeline
		rules:
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release' && $REF_COMMIT && $CUT_COMMIT

		test-be-to-release-stereo:
		stage: test
		tags:
		- ivas-windows
		- test-be-to-release-windows
		resource_group: ivas-be-to-release-test-resource
		timeout: "20 minutes"
		rules:
		- if: $CI_PIPELINE_SOURCE == 'web' && $MANUAL_PIPELINE_TYPE == 'test-be-release'
		timeout: "22 hours" # to be reviewed
		extends:
		- .rules-test-be-release
		script:
		- *comparison-setup-codec-ref-vs-cut
		- mkdir logs

		- echo "$CI_COMMIT_BRANCH"
		- MSBuild.exe .\Workspace_msvc\Workspace_msvc.sln /property:Configuration=Debug /p:Platform=win32 /m \| tee -variable winoutdata
		- $winoutdata \| Out-File $BUILD_OUTPUT -Encoding Utf8
		- ("& python ci/check_for_warnings.py '$BUILD_OUTPUT'") \| Invoke-Expression
		- ("exit $LASTEXITCODE") \| Invoke-Expression
		- mkdir tests\codec_be_to_accepted_release\ref
		- mkdir tests\codec_be_to_accepted_release\dut

		# path to release candidate refs defined in config.toml
		- echo "generate references"

		- echo "Placeholder for BE test to release script"
		- mkdir logs
		# use ref hrtf binaries
		- cp ref_hrtf_bin/* scripts/binauralRenderer_interface/binaural_renderers_hrtf_data

		# ref encoder stereo
		- $zero_errors_test_enc_const_br=0
		- $zero_failures_test_enc_const_br=0

		- ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_enc_const_br_stereo.py --ref_encoder_path ./IVAS_cod_ref.exe --update_ref 1 --reduction_level reduced --html=logs/report-test_enc_const_br_stereo_ref.html --self-contained-html --junit-xml=logs/report-junit-test_enc_const_br_stereo_ref.xml --testcase_timeout 360 ") \| Invoke-Expression

		- IF (!(Select-String logs/report-junit-test_enc_const_br_stereo_ref.xml -Pattern 'errors="0"')){$zero_errors_test_enc_const_br=1; echo "Errors encountered in stereo enc ref generation"}
		- IF (!(Select-String logs/report-junit-test_enc_const_br_stereo_ref.xml -Pattern 'failures="0"')){$zero_failures_test_enc_const_br=1; echo "Failed tests encountered in stereo enc ref generation"}

		# ref decoder stereo
		- $zero_errors_test_dec_const_br=0
		- $zero_failures_test_dec_const_br=0

		- ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_dec_const_br_stereo.py --ref_decoder_path ./IVAS_dec_ref.exe --update_ref 1 --reduction_level reduced --html=logs/report-test_dec_const_br_stereo_stereo_ref.html --self-contained-html --junit-xml=logs/report-junit-test_dec_const_br_stereo_stereo_ref.xml --testcase_timeout 360 ") \| Invoke-Expression

		- IF (!(Select-String logs/report-junit-test_dec_const_br_stereo_stereo_ref.xml -Pattern 'errors="0"')){$zero_errors_test_dec_const_br=1; echo "Errors encountered in stereo_stereo dec ref generation"}
		- IF (!(Select-String logs/report-junit-test_dec_const_br_stereo_stereo_ref.xml -Pattern 'failures="0"')){$zero_failures_test_dec_const_br=1; echo "Failed tests encountered in stereo_stereo dec ref generation"}

		# use cut hrtf binaries
		- cp cut_hrtf_bin/* scripts/binauralRenderer_interface/binaural_renderers_hrtf_data

		# cut encoder stereo
		- echo "bitstream comparison"
		- ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_enc_const_br_stereo.py --dut_encoder_path ./IVAS_cod.exe --reduction_level reduced --html=logs/report-test_enc_const_br_stereo_cut.html --self-contained-html --junit-xml=logs/report-junit-test_enc_const_br_stereo_cut.xml --testcase_timeout 360 ") \| Invoke-Expression

		- IF (!(Select-String logs/report-junit-test_enc_const_br_stereo_cut.xml -Pattern 'errors="0"')){$zero_errors_test_enc_const_br=1; echo "Errors encountered in stereo"}
		- IF (!(Select-String logs/report-junit-test_enc_const_br_stereo_cut.xml -Pattern 'failures="0"')){$zero_failures_test_enc_const_br=1; echo "Failed tests encountered in stereo ref generation"}

		# cut decoder stereo
		- echo "decoder output comparison \| cut decoder stereo"
		- ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_dec_const_br_stereo.py --dut_decoder_path ./IVAS_dec.exe --reduction_level reduced --html=logs/report-test_dec_const_br_stereo_stereo_cut.html --self-contained-html --junit-xml=logs/report-junit-test_dec_const_br_stereo_stereo_cut.xml --testcase_timeout 360 ") \| Invoke-Expression

		- IF (!(Select-String logs/report-junit-test_dec_const_br_stereo_stereo_cut.xml -Pattern 'errors="0"')){$zero_errors_test_dec_const_br=1; echo "Errors encountered in stereo_stereo dec cut generation"}
		- IF (!(Select-String logs/report-junit-test_dec_const_br_stereo_stereo_cut.xml -Pattern 'failures="0"')){$zero_failures_test_dec_const_br=1; echo "Failed tests encountered in stereo_stereo dec cut generation"}

		- IF($zero_errors_test_enc_const_br -ne 0 -or $zero_errors_test_dec_const_br -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL}
		- IF($zero_failures_test_enc_const_br -ne 0 -or $zero_failures_test_dec_const_br -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL}

		artifacts:
		name: "test-be-to-release--sha-$CI_COMMIT_SHORT_SHA--results"
		name: "test-be-to-release-stereo--sha-$CI_COMMIT_SHORT_SHA--results"
		when: always
		expire_in: 1 week
		paths:
		- logs/
		expose_as: "test-be-to-release results"
		expose_as: "test-be-to-release stereo results"
		reports:
		junit:
		- logs/report-junit-*.xml

		test-be-to-release-multichannel:
		stage: test
		tags:
		- test-be-to-release-windows
		resource_group: ivas-be-to-release-test-resource
		timeout: "22 hours" # to be reviewed
		extends:
		- .rules-test-be-release
		script:
		- *comparison-setup-codec-ref-vs-cut
		- mkdir logs

		- mkdir tests\codec_be_to_accepted_release\ref
		- mkdir tests\codec_be_to_accepted_release\dut

		- echo "generate references"

		# use ref hrtf binaries
		- cp ref_hrtf_bin/* scripts/binauralRenderer_interface/binaural_renderers_hrtf_data

		# ref encoder multichannel
		- $zero_errors_test_enc_const_br=0
		- $zero_failures_test_enc_const_br=0

		- ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_enc_const_br_multichannel.py --ref_encoder_path ./IVAS_cod_ref.exe --update_ref 1 --reduction_level reduced --html=logs/report-test_enc_const_br_multichannel_ref.html --self-contained-html --junit-xml=logs/report-junit-test_enc_const_br_multichannel_ref.xml --testcase_timeout 360 ") \| Invoke-Expression

		- IF (!(Select-String logs/report-junit-test_enc_const_br_multichannel_ref.xml -Pattern 'errors="0"')){$zero_errors_test_enc_const_br=1; echo "Errors encountered in multichannel enc ref generation"}
		- IF (!(Select-String logs/report-junit-test_enc_const_br_multichannel_ref.xml -Pattern 'failures="0"')){$zero_failures_test_enc_const_br=1; echo "Failed tests encountered in multichannel enc ref generation"}

		# ref decoder multichannel
		- $zero_errors_test_dec_const_br=0
		- $zero_failures_test_dec_const_br=0

		- ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_dec_const_br_multichannel.py --ref_decoder_path ./IVAS_dec_ref.exe --update_ref 1 --reduction_level reduced --html=logs/report-test_dec_const_br_multichannel_multichannel_ref.html --self-contained-html --junit-xml=logs/report-junit-test_dec_const_br_multichannel_multichannel_ref.xml --testcase_timeout 360 ") \| Invoke-Expression

		- IF (!(Select-String logs/report-junit-test_dec_const_br_multichannel_multichannel_ref.xml -Pattern 'errors="0"')){$zero_errors_test_dec_const_br=1; echo "Errors encountered in multichannel_multichannel dec ref generation"}
		- IF (!(Select-String logs/report-junit-test_dec_const_br_multichannel_multichannel_ref.xml -Pattern 'failures="0"')){$zero_failures_test_dec_const_br=1; echo "Failed tests encountered in multichannel_multichannel dec ref generation"}

		# use cut hrtf binaries
		- cp cut_hrtf_bin/* scripts/binauralRenderer_interface/binaural_renderers_hrtf_data

		# cut encoder multichannel
		- echo "cut encoder multichannel"
		- ("& python -m pytest .\tests\codec_be_to_accepted_release\encoder\test_enc_const_br_multichannel.py --dut_encoder_path ./IVAS_cod.exe --reduction_level reduced --html=logs/report-test_enc_const_br_multichannel_cut.html --self-contained-html --junit-xml=logs/report-junit-test_enc_const_br_multichannel_cut.xml --testcase_timeout 360 ") \| Invoke-Expression

		- IF (!(Select-String logs/report-junit-test_enc_const_br_multichannel_cut.xml -Pattern 'errors="0"')){$zero_errors_test_enc_const_br=1; echo "Errors encountered in multichannel"}
		- IF (!(Select-String logs/report-junit-test_enc_const_br_multichannel_cut.xml -Pattern 'failures="0"')){$zero_failures_test_enc_const_br=1; echo "Failed tests encountered in multichannel ref generation"}

		# cut decoder multichannel
		- echo "cut decoder multichannel"
		- ("& python -m pytest .\tests\codec_be_to_accepted_release\decoder\test_dec_const_br_multichannel.py --dut_decoder_path ./IVAS_dec.exe --reduction_level reduced --html=logs/report-test_dec_const_br_multichannel_multichannel_cut.html --self-contained-html --junit-xml=logs/report-junit-test_dec_const_br_multichannel_multichannel_cut.xml --testcase_timeout 360 ") \| Invoke-Expression

		- IF (!(Select-String logs/report-junit-test_dec_const_br_multichannel_multichannel_cut.xml -Pattern 'errors="0"')){$zero_errors_test_dec_const_br=1; echo "Errors encountered in multichannel_multichannel dec cut generation"}
		- IF (!(Select-String logs/report-junit-test_dec_const_br_multichannel_multichannel_cut.xml -Pattern 'failures="0"')){$zero_failures_test_dec_const_br=1; echo "Failed tests encountered in multichannel_multichannel dec cut generation"}

		- IF($zero_errors_test_enc_const_br -ne 0 -or $zero_errors_test_dec_const_br -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL}
		- IF($zero_failures_test_enc_const_br -ne 0 -or $zero_failures_test_dec_const_br -ne 0 ) {echo "Errors encountered"; exit $EXIT_CODE_FAIL}

		artifacts:
		name: "test-be-to-release-multichannel--sha-$CI_COMMIT_SHORT_SHA--results"
		when: always
		expire_in: 1 week
		paths:
		- logs/
		expose_as: "test-be-to-release multichannel results"
		reports:
		junit:
		- logs/report-junit-*.xml

		test-long-self-test:
		tags:

scripts/tools/Linux/eid-xor

0 → 100755

+29.8 KiB

File added.

No diff preview for this file type.

View file

scripts/tools/Win32/eid-xor.exe

0 → 100755LFS

+130 B

File added.

No diff preview for this file type.

View file

tests/codec_be_to_accepted_release/decoder/init.py

+37 −4

Original line number	Diff line number	Diff line
		@@ -30,6 +30,7 @@ the United Nations Convention on Contracts on the International Sales of Goods.

		import numpy as np
		import pytest
		import platform
		from contextlib import contextmanager
		from typing import Union
		from pathlib import Path
		@@ -140,9 +141,25 @@ def get_bitstream_and_options(
		bitstream_out = Path(tmp_dir).joinpath(bitstream.stem + f".{processing}.192")
		if processing == BS_PROC_FER_15:
		ep_path = TESTV_PATH.joinpath("ltv_ep_015.192")

		# get the pathname to the eid-xor tool respecting the running platform
		if platform.system() == "Windows":
		eid_xor_binary = [
		HERE.joinpath(
		"../../scripts/tools/Win32/eid-xor.exe"
		)
		]
		elif platform.system() in ["Linux", "Darwin"]:
		eid_xor_binary = [
		HERE.joinpath(
		"../../scripts/tools/Linux/eid-xor"
		)
		]
		else:
		assert False, f"eid-xor not available for {platform.system()}"

		cmd = [
		# TODO: adapt for windows
		"eid-xor",
		str(eid_xor_binary),
		"-fer",
		"-vbr",
		str(bitstream),
		@@ -156,10 +173,26 @@ def get_bitstream_and_options(
		"../../scripts/dly_error_profiles/dly_error_profile_5.dat"
		)
		tracefile_path = Path(tmp_dir).joinpath(bitstream_out.with_suffix(".trace"))
		# TODO: adapt for windows

		# get the pathname to the networkSimulator_g192 tool respecting the running platform
		if platform.system() == "Windows":
		networkSimulator_g192_binary = [
		HERE.joinpath(
		"../../scripts/tools/Win32/networkSimulator_g192.exe"
		)
		]
		elif platform.system() in ["Linux", "Darwin"]:
		networkSimulator_g192_binary = [
		HERE.joinpath(
		"../../scripts/tools/Linux/networkSimulator_g192"
		)
		]
		else:
		assert False, f"networkSimulator_g192 not available for {platform.system()}"

		# TODO: get number of frames per packet from error profile name
		cmd = [
		"networkSimulator_g192",
		str(networkSimulator_g192_binary),
		str(dly_profile_path),
		str(bitstream),
		str(bitstream_out),