Merge branch 'ci/be-test-5ms' into 'main' (410b27b9) · Commits · IVAS Codec Public Collaboration / IVAS Codec

.gitlab-ci.yml

+54 −0

Original line number	Diff line number	Diff line
		@@ -167,6 +167,10 @@ stages:
		- sed -i.bak -e "s/\/\[[:space:]]$#define[[:space:]]SPLIT_REND_WITH_HEAD_ROT$[[:space:]]\*\//\1/g" ./lib_com/options.h
		- sed -i.bak -e "s/\/\/[[:space:]]$#define[[:space:]]SPLIT_REND_WITH_HEAD_ROT$/\1/g" ./lib_com/options.h

		.disable-limiter: &disable-limiter
		# automatically enable #define DISABLE_LIMITER in options.h, handling both /**/-comment and //-comment
		- sed -i.bak -e "s/\/\[[:space:]]$#define[[:space:]]DISABLE_LIMITER$[[:space:]]\*\//\1/g" ./lib_com/options.h

		.get-commits-behind-count: &get-commits-behind-count
		- echo $CI_COMMIT_SHA
		- echo $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
		@@ -523,6 +527,54 @@ codec-usan:
		- SELF_TEST_PRM_FILE="tests/codec_be_on_mr_nonselection/test_param_file.py"
		<<: *sanitizer-selftest-anchor

		# compare bit-exactness between 5ms and 20 on the branch
		pytest-compare-20ms-and-5ms-rendering:
		extends:
		- .test-job-linux
		- .rules-merge-request
		stage: test
		needs: ["build-codec-linux-cmake", "build-codec-linux-make", "build-codec-instrumented-linux", "build-codec-sanitizers-linux"]
		script:
		- *print-common-info
		- *disable-limiter
		- make clean
		- make -j
		### prepare pytest
		- cp IVAS_cod IVAS_cod_ref
		- cp IVAS_dec IVAS_dec_ref
		# create short test vectors
		- python3 tests/create_short_testvectors.py
		# create references
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --update_ref 1 -m create_ref_part2
		### run pytest
		- exit_code=0
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-5ms.html --self-contained-html --junit-xml=report-junit-5ms.xml --dut_fr 5 --decoder_only \|\| exit_code5=$?
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --html=report-10ms.html --self-contained-html --junit-xml=report-junit-10ms.xml --dut_fr 10 --decoder_only \|\| exit_code10=$?
		- zero_errors5=$(cat report-junit-5ms.xml \| grep -c 'errors="0"') \|\| true
		- zero_errors10=$(cat report-junit-10ms.xml \| grep -c 'errors="0"') \|\| true
		- zero_errors=1
		- if [ $zero_errors5 != 1 ]; then echo "run error in with 5ms rendering encountered"; zero_errors=0 ; fi
		- if [ $zero_errors10 != 1 ]; then echo "run error in with 10ms rendering encountered"; zero_errors=0 ; fi
		- if [ $zero_errors != 1 ]; then exit $EXIT_CODE_FAIL; fi
		- if [ $exit_code5 -eq 1 ]; then echo "Non-bitexact cases encountered with 5ms rendering!"; exit_code=1; fi
		- if [ $exit_code10 -eq 1 ]; then echo "Non-bitexact cases encountered with 10ms rendering!"; exit_code=1; fi
		- if [ $exit_code -eq 1 ]; then exit $EXIT_CODE_FAIL; fi
		artifacts:
		name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"
		expire_in: 1 week
		when: always
		expose_as: "pytest 5ms and 10ms vs 20ms results"
		paths:
		- report-junit-5ms.xml
		- report-5ms.html
		- report-junit-10ms.xml
		- report-10ms.html
		reports:
		junit:
		- report-junit-5ms.xml
		- report-junit-10ms.xml

		# test renderer executable
		renderer-smoke-test:
		extends:
		@@ -1988,6 +2040,8 @@ coverage-test-on-main-scheduled:
		- python3 tests/create_short_testvectors.py
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 -m create_ref --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v -n auto --update_ref 1 -m create_ref_part2 --ref_encoder_path ./IVAS_cod --ref_decoder_path ./IVAS_dec
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 5 --decoder_only
		- python3 -m pytest $TESTS_DIR_CODEC_BE_ON_MR -v --dut_encoder_path ./IVAS_cod --dut_decoder_path ./IVAS_dec --dut_fr 10 --decoder_only
		- python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_ref
		- python3 -m pytest -q -n auto tests/renderer/test_renderer.py --create_cut
		- lcov -c -d obj -o coverage_stv.info # extract coverage of short test vectors here

lib_com/options.h

+1 −0

Original line number	Diff line number	Diff line
		@@ -146,6 +146,7 @@

		/* keep as part of options.h */
		#define BASOP_NOGLOB /* Disable global symbols in BASOPs, Overflow/Carry in BASOPs disabled, additional BASOPs in case of Overflow */
		/#define DISABLE_LIMITER / /* test switch for testing BE between 5ms and 20ms rendering */

		/* ################## Start DEVELOPMENT switches ######################### */

scripts/config/self_test.prm

+0 −20

Original line number	Diff line number	Diff line
		@@ -685,10 +685,6 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1
		../IVAS_cod -sba 3 48000 48 testv/stv3OA48c.wav bit
		../IVAS_dec 5_1_2 48 bit testv/stv3OA48c.wav_SBA_48000_48-48_5_1_2.tst

		// SBA at 48 kbps, 48kHz in, 48kHz out, BINAURAL out, fr 5ms
		../IVAS_cod -sba 3 48000 48 testv/stv3OA48c.wav bit
		../IVAS_dec -fr 5 -t testv/headrot.csv BINAURAL 48 bit testv/stv3OA48c.wav_SBA_48000_48-48_binaural_ht_fr5.tst

		// SBA at 64 kbps, 32kHz in, 32kHz out, FOA out, DTX, random FER at 5%
		../IVAS_cod -dtx -sba 1 64000 32 testv/stvFOA32c.wav bit
		eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g192 bit_error
		@@ -806,10 +802,6 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1
		../IVAS_cod -sba 3 512000 48 testv/stv3OA48c.wav bit
		../IVAS_dec -t testv/headrot.csv -exof testv/headrot_case00_3000_q_combinedRotationTest.csv BINAURAL 48 bit testv/stv3OA48c.wav_SBA_512000_48-48_Binaural_Headrot_EXOF.tst

		// SBA at 512 kbps, 48kHz in, 48kHz out, BINAURAL out, fr 5ms
		../IVAS_cod -sba 3 512000 48 testv/stv3OA48c.wav bit
		../IVAS_dec -fr 5 -t testv/headrot.csv BINAURAL 48 bit testv/stv3OA48c.wav_SBA_512000_48-48_binaural_ht_fr5.tst

		// SBA FOA bitrate switching from 13.2 kbps to 192 kbps, 48kHz in, 48kHz out, BINAURAL out
		../IVAS_cod -sba 1 ../scripts/switchPaths/sw_13k2_192k_50fr.bin 48 testv/stvFOA48c.wav bit
		../IVAS_dec BINAURAL 48 bit testv/stvFOA48c.wav_sw_48-48_BINAURAL.tst
		@@ -1236,18 +1228,6 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1
		../IVAS_cod -mc 7_1_4 160000 48 testv/stv714MC48c.wav bit
		../IVAS_dec 7_1_4 48 bit testv/stv714MC48c.wav_MC714_160000_48-48_MC714.tst

		// Multi-channel 7_1_4 at 160 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM out, fr 5ms
		../IVAS_cod -mc 7_1_4 160000 48 testv/stv714MC48c.wav bit
		../IVAS_dec -fr 5 BINAURAL_ROOM_IR 48 bit testv/stv714MC48c.wav_MC714_160000_48-48_MC_binaural_room_fr5.tst

		// Multi-channel 7_1_4 at 512 kbps, 48kHz in, 48kHz out, with headtracking, BINAURAL out, fr 5ms
		../IVAS_cod -mc 7_1_4 512000 48 testv/stv714MC48c.wav bit
		../IVAS_dec -fr 5 -t testv/headrot.csv BINAURAL 48 bit testv/stv714MC48c.wav_MC714_512000_48-48_MC_binaural_ht_fr5.tst

		// Multi-channel 5_1 at 512 kbps, 48kHz in, 48kHz out, BINAURAL out, fr 5ms
		../IVAS_cod -mc 5_1 512000 48 testv/stv51MC48c.wav bit
		../IVAS_dec -fr 5 -t testv/headrot.csv BINAURAL 48 bit testv/stv51MC48c.wav_MC51_512000_48-48_MC_binaural_ht_fr5.tst

		// Multi-channel 5_1_2 at 32 kbps, 48kHz in, 48kHz out, STEREO out, random FER at 5%
		../IVAS_cod -mc 5_1_2 32000 48 testv/stv512MC48c.wav bit
		eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g192 bit_error

scripts/config/self_test_ltv.prm

+0 −20

Original line number	Diff line number	Diff line
		@@ -685,10 +685,6 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1
		../IVAS_cod -sba 3 48000 48 testv/ltv48_HOA3.wav bit
		../IVAS_dec 5_1_2 48 bit testv/ltv48_HOA3.wav_SBA_48000_48-48_5_1_2.tst

		// SBA at 48 kbps, 48kHz in, 48kHz out, BINAURAL out, fr 5ms
		../IVAS_cod -sba 3 48000 48 testv/ltv48_HOA3.wav bit
		../IVAS_dec -fr 5 -t testv/headrot.csv BINAURAL 48 bit testv/ltv48_HOA3.wav_SBA_48000_48-48_binaural_ht_fr5.tst

		// SBA at 64 kbps, 32kHz in, 32kHz out, FOA out, DTX, random FER at 5%
		../IVAS_cod -dtx -sba 1 64000 32 testv/ltv32_FOA.wav bit
		eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g192 bit_error
		@@ -806,10 +802,6 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1
		../IVAS_cod -sba 3 512000 48 testv/ltv48_HOA3.wav bit
		../IVAS_dec -t testv/headrot.csv -exof testv/headrot_case00_3000_q_combinedRotationTest.csv BINAURAL 48 bit testv/ltv48_HOA3.wav_SBA_512000_48-48_Binaural_Headrot_EXOF.tst

		// SBA at 512 kbps, 48kHz in, 48kHz out, BINAURAL out, fr 5ms
		../IVAS_cod -sba 3 512000 48 testv/ltv48_HOA3.wav bit
		../IVAS_dec -fr 5 -t testv/headrot.csv BINAURAL 48 bit testv/ltv48_HOA3.wav_SBA_512000_48-48_binaural_ht_fr5.tst

		// SBA FOA bitrate switching from 13.2 kbps to 192 kbps, 48kHz in, 48kHz out, BINAURAL out
		../IVAS_cod -sba 1 ../scripts/switchPaths/sw_13k2_192k_50fr.bin 48 testv/ltv48_FOA.wav bit
		../IVAS_dec BINAURAL 48 bit testv/ltv48_FOA.wav_sw_48-48_BINAURAL.tst
		@@ -1237,18 +1229,6 @@ eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g1
		../IVAS_cod -mc 7_1_4 160000 48 testv/ltv48_MC714.wav bit
		../IVAS_dec 7_1_4 48 bit testv/ltv48_MC714.wav_MC714_160000_48-48_MC714.tst

		// Multi-channel 7_1_4 at 160 kbps, 48kHz in, 48kHz out, BINAURAL_ROOM out, fr 5ms
		../IVAS_cod -mc 7_1_4 160000 48 testv/ltv48_MC714.wav bit
		../IVAS_dec -fr 5 BINAURAL_ROOM_IR 48 bit testv/ltv48_MC714.wav_MC714_160000_48-48_MC_binaural_room_fr5.tst

		// Multi-channel 7_1_4 at 512 kbps, 48kHz in, 48kHz out, with headtracking, BINAURAL out, fr 5ms
		../IVAS_cod -mc 7_1_4 512000 48 testv/ltv48_MC714.wav bit
		../IVAS_dec -fr 5 -t testv/headrot.csv BINAURAL 48 bit testv/ltv48_MC714.wav_MC714_512000_48-48_MC_binaural_ht_fr5.tst

		// Multi-channel 5_1 at 512 kbps, 48kHz in, 48kHz out, BINAURAL out, fr 5ms
		../IVAS_cod -mc 5_1 512000 48 testv/ltv48_MC51.wav bit
		../IVAS_dec -fr 5 -t testv/headrot.csv BINAURAL 48 bit testv/ltv48_MC714.wav_MC51_512000_48-48_MC_binaural_ht_fr5.tst

		// Multi-channel 5_1_2 at 32 kbps, 48kHz in, 48kHz out, STEREO out, random FER at 5%
		../IVAS_cod -mc 5_1_2 32000 48 testv/ltv48_MC512.wav bit
		eid-xor -fer -vbr -bs g192 -ep g192 bit ../scripts/dly_error_profiles/ep_5pct.g192 bit_error

scripts/self_test.py

+9 −0

Original line number	Diff line number	Diff line
		@@ -195,6 +195,13 @@ class SelfTest(IvasScriptsCommon.IvasScript):
		action="store_true",
		default=False,
		)
		self.parser.add_argument(
		"--dut_fr",
		help="Run the decoder under test with specified render framn size",
		default=20,
		choices=[5, 10, 20],
		type=int
		)
		if shutil.which("valgrind"):
		self.valgrind = [
		"valgrind",
		@@ -1520,6 +1527,8 @@ class SelfTest(IvasScriptsCommon.IvasScript):
		enable_logging=True,
		logger_name="{}.testrunner".format(self.logger.name),
		)

		test_runner.decoder_cmdline_options.extend(["-fr", f"{self.args['dut_fr']}"])
		test_runner.set_flat_mode_list(run_dict)
		test_runner.run()
		self.logger.console(" ")