Merge remote-tracking branch 'origin/main' into pajunen/remove-empty-headrotation-file-reader (5162d30a) · Commits · IVAS Codec Public Collaboration / IVAS Codec

.gitlab-ci.yml

+75 −15

Original line number	Diff line number	Diff line
		@@ -10,6 +10,7 @@ variables:
		OUT_FORMATS_BINAURAL: "BINAURAL BINAURAL_ROOM_IR BINAURAL_ROOM_REVERB"
		EXIT_CODE_NON_BE: 123
		EXIT_CODE_FAIL: 1
		PROCESSING_SCRIPTS_BIN_DIR: "/test-bin"

		default:
		interruptible: true # Make all jobs by default interruptible
		@@ -69,7 +70,9 @@ stages:
		# compare to last target branch commit before pipeline was created
		- target_commit=$(git log $CI_MERGE_REQUEST_TARGET_BRANCH_NAME -1 --oneline --before=${CI_PIPELINE_CREATED_AT} --format=%H)


		.check-for-testvectors: &check-for-testvectors
		# check if the testvector files specified in scripts/config/ci_linux*.json are present
		- python3 -m pytest ci/test_vectors_available.py

		.merge-request-comparison-setup-codec:
		&merge-request-comparison-setup-codec ### build test binaries, initial clean for paranoia reasons
		@@ -283,7 +286,7 @@ build-codec-sanitizers-linux:
		build-codec-windows-cmake:
		extends:
		- .build-job-windows-with-check-for-warnings
		- .rules-basis
		- .rules-main-push
		script:
		- *print-common-info-windows
		- $winoutdata = $null
		@@ -296,7 +299,7 @@ build-codec-windows-cmake:
		build-codec-windows-msbuild:
		extends:
		- .build-job-windows-with-check-for-warnings
		- .rules-basis
		- .rules-main-push
		script:
		- *print-common-info-windows
		- $winoutdata = $null
		@@ -660,30 +663,74 @@ clang-format-check:
		expose_as: 'formatting patch'

		# check for crashes if first received frame on decoder side is an SID
		.check-first-frame-is-sid:
		check-first-frame-is-sid:
		extends:
		- .test-job-linux-needs-testv-dir
		- .rules-merge-request
		tags:
		- ivas-linux
		stage: test
		needs: ["build-codec-linux-cmake"]
		# needs: ["build-codec-linux-cmake"]
		script:
		- *print-common-info
		- *update-ltv-repo
		- *check-for-testvectors
		- cmake .
		- make -j

		# TODO: for some MASA modes, we currently do not have testvectors that actually trigger DTX
		- modes=$(scripts/runIvasCodec.py -l \| grep dtx \| grep -v MASA)
		- echo $modes
		- scripts/runIvasCodec.py -p scripts/config/ci_linux_sidstart_test.json -m $modes -s --bs_length 500
		# SBA is run separately to use shorter part of file
		- modes=$(scripts/runIvasCodec.py -l \| grep dtx \| grep -v MASA \| grep -v SBA)
		- scripts/runIvasCodec.py -p scripts/config/ci_linux_sidstart_test.json -m $modes -s --bs_length 500 -U 0:20 \|\| exit_code_no_sba=$?
		- modes=$(scripts/runIvasCodec.py -l \| grep dtx \| grep SBA)
		- scripts/runIvasCodec.py -p scripts/config/ci_linux_sidstart_test.json -m $modes -s --bs_length 500 -U 70:80 \|\| exit_code_sba=$?
		- if [ $exit_code_no_sba != 0 \|\| $exit_code_sba != 0]; then exit 1; fi
		artifacts:
		paths:
		- out/logs
		when: on_failure
		name: "$CI_JOB_NAME--$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--sidstart"
		expose_as: "logs-sidstart"
		expire_in: "5 days"

		selection-test-processing:
		extends:
		- .test-job-linux-needs-testv-dir
		- .rules-merge-request
		tags:
		- processing-scripts-linux
		stage: test
		needs: ["build-codec-linux-make"]
		script:
		# get processing script code
		- git clone https://forge.3gpp.org/rep/ivas-codec-pc/ivas-processing-scripts.git --single-branch -b main

		- mkdir -p ivas-processing-scripts/tests/data/testv
		- cp -r scripts/testv/* ivas-processing-scripts/tests/data/testv/

		# copy binaries into local bin dir, those should take precendence over PATH
		- cp $PROCESSING_SCRIPTS_BIN_DIR/* ivas-processing-scripts/ivas_processing_scripts/bin/
		# for testing with native binaries
		- rm ivas-processing-scripts/ivas_processing_scripts/bin/IVAS*.exe
		- rm ivas-processing-scripts/ivas_processing_scripts/bin/EVS*.exe

		# build codec and put into bin dir
		- make -j
		- cp ./IVAS_* ivas-processing-scripts/ivas_processing_scripts/bin/

		# patch the use_windows_codec_binaries key (weird folding is needed so colons are accepted)
		- >
		sed -i "s/use_windows_codec_binaries: true/use_windows_codec_binaries: false/" ivas-processing-scripts/experiments/selection//config/.yml

		# run experiments test
		- cd ivas-processing-scripts
		- python3 -m pytest tests/test_experiments.py::test_generate_test_items -n auto \| tee log.txt
		artifacts:
		paths:
		- ivas-processing-scripts/experiments/selection//proc_output/.log
		- log.txt
		when: on_failure
		expire_in: 1 week

		# ---------------------------------------------------------------
		# Test jobs for main branch
		@@ -906,7 +953,7 @@ sanitizer-test-ism3:
		- if: $SANITIZER_SCHEDULE_A
		when: delayed
		start_in: 11 hours
		timeout: 3 hour
		timeout: 4 hour
		script:
		- *update-ltv-repo
		- python3 ci/run_scheduled_sanitizer_test.py ISM3 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS
		@@ -916,8 +963,8 @@ sanitizer-test-ism4:
		rules:
		- if: $SANITIZER_SCHEDULE_A
		when: delayed
		start_in: 14 hours
		timeout: 4 hours
		start_in: 15 hours
		timeout: 6 hours
		script:
		- *update-ltv-repo
		- python3 ci/run_scheduled_sanitizer_test.py ISM4 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS
		@@ -927,7 +974,7 @@ sanitizer-test-masa:
		rules:
		- if: $SANITIZER_SCHEDULE_A
		when: delayed
		start_in: 18 hours
		start_in: 21 hours
		timeout: 3 hours
		script:
		- *update-ltv-repo
		@@ -1045,7 +1092,7 @@ sanitizer-test-ism+3:
		- if: $SANITIZER_SCHEDULE_D
		when: delayed
		start_in: 5 hours
		timeout: 3 hour
		timeout: 4 hours
		script:
		- *update-ltv-repo
		- python3 ci/run_scheduled_sanitizer_test.py ISM+3 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS
		@@ -1055,8 +1102,8 @@ sanitizer-test-ism+4:
		rules:
		- if: $SANITIZER_SCHEDULE_D
		when: delayed
		start_in: 8 hours
		timeout: 4 hours
		start_in: 9 hours
		timeout: 6 hours
		script:
		- *update-ltv-repo
		- python3 ci/run_scheduled_sanitizer_test.py ISM+4 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS
		@@ -1146,6 +1193,7 @@ coverage-test-on-main-scheduled:
		stage: test
		artifacts:
		name: "$CI_JOB_NAME--$CI_COMMIT_REF_NAME--sha-$CI_COMMIT_SHA"
		when: always
		expire_in: 1 week
		paths:
		- $CI_JOB_NAME-public
		@@ -1162,7 +1210,9 @@ complexity-stereo-in-stereo-out:
		- in_format=stereo
		- out_format=stereo
		- bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format"
		- ret_val=$?
		- *complexity-measurements-prepare-artifacts
		- exit $ret_val

		complexity-ism-in-binaural-out:
		extends:
		@@ -1178,7 +1228,9 @@ complexity-ism-in-binaural-out:
		- in_format=ISM
		- out_format=BINAURAL
		- bash ci/complexity_measurements/getWmops.sh "ISM+1 ISM+2 ISM+3 ISM+4" "$out_format"
		- ret_val=$?
		- *complexity-measurements-prepare-artifacts
		- exit $ret_val

		complexity-sba-hoa3-in-hoa3-out:
		extends:
		@@ -1194,7 +1246,9 @@ complexity-sba-hoa3-in-hoa3-out:
		- in_format=SBA
		- out_format=HOA3
		- bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format"
		- ret_val=$?
		- *complexity-measurements-prepare-artifacts
		- exit $ret_val

		complexity-mc-in-7_1_4-out:
		extends:
		@@ -1210,7 +1264,9 @@ complexity-mc-in-7_1_4-out:
		- in_format=MC
		- out_format=7_1_4
		- bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format"
		- ret_val=$?
		- *complexity-measurements-prepare-artifacts
		- exit $ret_val

		complexity-masa-in-7_1_4-out:
		extends:
		@@ -1226,7 +1282,9 @@ complexity-masa-in-7_1_4-out:
		- in_format=MASA
		- out_format=7_1_4
		- bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format"
		- ret_val=$?
		- *complexity-measurements-prepare-artifacts
		- exit $ret_val

		complexity-StereoDmxEVS-stereo-in-mono-out:
		extends:
		@@ -1242,7 +1300,9 @@ complexity-StereoDmxEVS-stereo-in-mono-out:
		- in_format=StereoDmxEVS
		- out_format=mono
		- bash ci/complexity_measurements/getWmops.sh "$in_format" "$out_format"
		- ret_val=$?
		- *complexity-measurements-prepare-artifacts
		- exit $ret_val

		# ---------------------------------------------------------------
		# Other jobs

apps/decoder.c

+20 −0

Original line number	Diff line number	Diff line
		@@ -1580,7 +1580,11 @@ static ivas_error initOnFirstGoodFrame(

		/* Duplicate good first frame metadata to fill the beginning of stream. */
		MASA_DECODER_EXT_OUT_META_HANDLE hMasaExtOutMeta = NULL;
		#ifdef FIX_470_MASA_JBM_EXT
		if ( ( error = IVAS_DEC_GetMasaMetadata( hIvasDec, &hMasaExtOutMeta, 0 ) ) != IVAS_ERR_OK )
		#else
		if ( ( error = IVAS_DEC_GetMasaMetadata( hIvasDec, &hMasaExtOutMeta ) ) != IVAS_ERR_OK )
		#endif
		{
		fprintf( stderr, "\nError in IVAS_DEC_GetMasaMetadata: %s\n", IVAS_DEC_GetErrorMessage( error ) );
		return error;
		@@ -1878,7 +1882,11 @@ static ivas_error decodeG192(
		else if ( bsFormat == IVAS_DEC_BS_MASA )
		{
		MASA_DECODER_EXT_OUT_META_HANDLE hMasaExtOutMeta;
		#ifdef FIX_470_MASA_JBM_EXT
		if ( ( error = IVAS_DEC_GetMasaMetadata( hIvasDec, &hMasaExtOutMeta, 0 ) ) != IVAS_ERR_OK )
		#else
		if ( ( error = IVAS_DEC_GetMasaMetadata( hIvasDec, &hMasaExtOutMeta ) ) != IVAS_ERR_OK )
		#endif
		{
		fprintf( stderr, "\nError in IVAS_DEC_GetMasaMetadata: %s\n", IVAS_DEC_GetErrorMessage( error ) );
		goto cleanup;
		@@ -2426,7 +2434,11 @@ static ivas_error decodeVoIP(
		else if ( bsFormat == IVAS_DEC_BS_MASA )
		{
		MASA_DECODER_EXT_OUT_META_HANDLE hMasaExtOutMeta;
		#ifdef FIX_470_MASA_JBM_EXT
		if ( ( error = IVAS_DEC_GetMasaMetadata( hIvasDec, &hMasaExtOutMeta, 1 ) ) != IVAS_ERR_OK )
		#else
		if ( ( error = IVAS_DEC_GetMasaMetadata( hIvasDec, &hMasaExtOutMeta ) ) != IVAS_ERR_OK )
		#endif
		{
		fprintf( stderr, "\nError in IVAS_DEC_GetMasaMetadata: %s\n", IVAS_DEC_GetErrorMessage( error ) );
		goto cleanup;
		@@ -2838,7 +2850,11 @@ static ivas_error decodeVariableSpeed(
		else if ( bsFormat == IVAS_DEC_BS_MASA )
		{
		MASA_DECODER_EXT_OUT_META_HANDLE hMasaExtOutMeta;
		#ifdef FIX_470_MASA_JBM_EXT
		if ( ( error = IVAS_DEC_GetMasaMetadata( hIvasDec, &hMasaExtOutMeta, 1 ) ) != IVAS_ERR_OK )
		#else
		if ( ( error = IVAS_DEC_GetMasaMetadata( hIvasDec, &hMasaExtOutMeta ) ) != IVAS_ERR_OK )
		#endif
		{
		fprintf( stderr, "\nError in IVAS_DEC_GetMasaMetadata: %s\n", IVAS_DEC_GetErrorMessage( error ) );
		goto cleanup;
		@@ -3008,7 +3024,11 @@ static ivas_error decodeVariableSpeed(
		else if ( bsFormat == IVAS_DEC_BS_MASA )
		{
		MASA_DECODER_EXT_OUT_META_HANDLE hMasaExtOutMeta;
		#ifdef FIX_470_MASA_JBM_EXT
		if ( ( error = IVAS_DEC_GetMasaMetadata( hIvasDec, &hMasaExtOutMeta, 0 ) ) != IVAS_ERR_OK )
		#else
		if ( ( error = IVAS_DEC_GetMasaMetadata( hIvasDec, &hMasaExtOutMeta ) ) != IVAS_ERR_OK )
		#endif
		{
		fprintf( stderr, "\nError in IVAS_DEC_GetMasaMetadata: %s\n", IVAS_DEC_GetErrorMessage( error ) );
		goto cleanup;

ci/complexity_measurements/getWmops.sh

+4 −1

Original line number	Diff line number	Diff line
		@@ -53,8 +53,11 @@ config_file="scripts/config/ci_linux_ltv.json"
		wmopsFilenameFlcLast=wmops_newsletter_stereo__${commit_sha}_${date}
		wmopsFilenameFlc=${destDir}/wmops/logs/${wmopsFilenameFlcLast}

		ret_val=0

		# instrument and build
		./scripts/IvasBuildAndRunChecks.py -p $config_file --checks COMPLEXITY --create_complexity_tables ${wmopsFilenameFlc} -C $ivas_format -f ${ep} --oc $output_format
		ret_val=$?

		# get the info on worst-case operating point: WMOPS number, enc-operating mode, dec-operating mode
		### WMOPS
		@@ -83,4 +86,4 @@ ${scriptDir}/parseNewsletterRom.py ${wmopsFilenameFlc}_PROM.csv ${wmopsFilenameF
		# generate java script from database
		tcsh ${scriptDir}/genWebpageData_Rom.csh ${destDir}/wmops/log_rom_all.txt ${destDir}/wmops/graphs_rom_flc.js Graphs_ROM


		exit $ret_val
		No newline at end of file

ci/test_vectors_available.py

0 → 100644

+19 −0

Original line number	Diff line number	Diff line
		import pytest
		import pathlib
		import json
		import itertools

		TEST_CONFIG_DIR = pathlib.Path(__file__).parent.parent.joinpath("scripts/config")
		TEST_CONFIGS = [f for f in TEST_CONFIG_DIR.iterdir() if f.name.startswith("ci_linux")]

		def get_testvectors_from_config(config) -> list:
		with open(config) as f:
		cfg = json.load(f)
		return list(cfg["inpaths"].values())

		TESTVECTORS = sorted(set(itertools.chain(*[get_testvectors_from_config(cfg) for cfg in TEST_CONFIGS])))

		@pytest.mark.parametrize("testvector", TESTVECTORS)
		def test_vectors_available(testvector):
		if not pathlib.Path(testvector).exists():
		raise FileNotFoundError(f"Testvector {testvector} can not be found")

lib_com/bitstream.c

100755 → 100644

+4 −329

File changed.File mode changed from 100755 to 100644.

Preview size limit exceeded, changes collapsed.