Merge branch 'main' into 112-add-self_test_evs-prm-to-mr-pipeline-not-allowed-to-fail (02095a10) · Commits · IVAS Codec Public Collaboration / IVAS Codec

.gitlab-ci.yml

+193 −18

Original line number	Diff line number	Diff line
		@@ -2,6 +2,10 @@ variables:
		TESTV_DIR: "/usr/local/testv"
		BUILD_OUTPUT: "build_output.txt"
		EVS_BE_TEST_DIR: "/usr/local/be_2_evs_test"
		SANITIZER_TESTS: "CLANG1 CLANG2"
		OUT_FORMATS_CHANNEL_BASED: "stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4"
		OUT_FORMATS_SCENE_BASED: "FOA HOA2 HOA3"
		OUT_FORMATS_BINAURAL: "BINAURAL BINAURAL_ROOM"


		# This sets when pipelines are created. Jobs have more specific rules to restrict them.
		@@ -14,7 +18,6 @@ workflow:
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main
		- if: $CI_PIPELINE_SOURCE == 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Scheduled in main


		stages:
		- maintenance
		- build
		@@ -480,34 +483,206 @@ codec-comparison-on-main-push:
		junit: report-junit.xml


		# parameterizable job for sanitizer tests per format
		# how to set up: create a schedule (CI/CD -> schedules) and enter the respective values for the environment variables:
		# - SANITIZER_TEST_IN_FMT: input format
		# - SANITIZER_TEST_OUT_FMTS: list of output formats, blank-separated, e.g.: stereo mono 5_1
		# - SANITIZER_TEST_TESTS: list of checks to do, can be one of CLANG1, CLANG2, CLANG3, VALGRIND
		sanitizer-test-on-main-scheduled:
		# ---------------------------------------------------------------
		# Scheduled jobs on main
		# ---------------------------------------------------------------
		.sanitizer-test-template:
		extends:
		- .test-job-linux-needs-testv-dir
		# this next one is maybe not really needed, since there is the rule checking for the existence of the env vars below, but use for clarity
		- .rules-main-scheduled
		stage: test
		tags:
		- sanitizer_test_main

		sanitizer-test-mono:
		extends: .sanitizer-test-template
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		script:
		- python3 ci/run_scheduled_sanitizer_test.py mono mono --tests $SANITIZER_TESTS

		sanitizer-test-stereo:
		extends: .sanitizer-test-template
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		when: delayed
		start_in: 20 minutes
		script:
		- python3 ci/run_scheduled_sanitizer_test.py stereo $OUT_FORMATS_CHANNEL_BASED --tests $SANITIZER_TESTS

		sanitizer-test-stereodmxevs:
		extends: .sanitizer-test-template
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		when: delayed
		start_in: 40 minutes
		script:
		- python3 ci/run_scheduled_sanitizer_test.py StereoDmxEvs mono --tests $SANITIZER_TESTS

		sanitizer-test-ism1:
		extends: .sanitizer-test-template
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		when: delayed
		start_in: 1 hour
		script:
		- python3 ci/run_scheduled_sanitizer_test.py ISM1 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS

		sanitizer-test-ism2:
		extends: .sanitizer-test-template
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		when: delayed
		start_in: 1 hour 30 minutes
		script:
		- python3 ci/run_scheduled_sanitizer_test.py ISM2 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS

		sanitizer-test-ism3:
		extends: .sanitizer-test-template
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		when: delayed
		start_in: 2 hours
		script:
		- python3 ci/run_scheduled_sanitizer_test.py ISM3 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS

		sanitizer-test-ism4:
		extends: .sanitizer-test-template
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		when: delayed
		start_in: 2 hours 30 minutes
		script:
		- python3 ci/run_scheduled_sanitizer_test.py ISM4 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS

		sanitizer-test-mc-5_1:
		extends:
		- .test-job-linux-needs-testv-dir
		stage: test
		tags:
		- sanitizer_test_main
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		when: delayed
		start_in: 3 hours
		script:
		- python3 ci/run_scheduled_sanitizer_test.py 5_1 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS

		sanitizer-test-mc-5_1_2:
		extends:
		- .test-job-linux-needs-testv-dir
		stage: test
		tags:
		- sanitizer_test_main
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		when: delayed
		start_in: 4 hours
		script:
		- python3 ci/run_scheduled_sanitizer_test.py 5_1_2 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS

		sanitizer-test-mc-5_1_4:
		extends:
		- .test-job-linux-needs-testv-dir
		stage: test
		tags:
		- sanitizer_test_main
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		when: delayed
		start_in: 5 hours
		script:
		- python3 ci/run_scheduled_sanitizer_test.py 5_1_4 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS

		sanitizer-test-mc-7_1:
		extends:
		- .test-job-linux-needs-testv-dir
		stage: test
		tags:
		- sanitizer_test_main
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		when: delayed
		start_in: 6 hours
		script:
		- python3 ci/run_scheduled_sanitizer_test.py 7_1 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS

		sanitizer-test-mc-7_1_4:
		extends:
		- .test-job-linux-needs-testv-dir
		stage: test
		tags:
		- sanitizer_test_main
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		when: delayed
		start_in: 7 hours
		script:
		- python3 ci/run_scheduled_sanitizer_test.py 7_1_4 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS

		sanitizer-test-masa:
		extends:
		- .test-job-linux-needs-testv-dir
		stage: test
		tags:
		- sanitizer_test_main
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		when: delayed
		start_in: 8 hours
		script:
		- python3 ci/run_scheduled_sanitizer_test.py MASA $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS

		sanitizer-test-sba:
		extends:
		- .test-job-linux-needs-testv-dir
		stage: test
		tags:
		- sanitizer_test_main
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		when: delayed
		start_in: 9 hours
		script:
		- python3 ci/run_scheduled_sanitizer_test.py SBA $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS

		sanitizer-test-planarsba:
		extends:
		- .test-job-linux-needs-testv-dir
		stage: test
		tags:
		- sanitizer_test_main
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		when: delayed
		start_in: 10 hours
		script:
		- python3 ci/run_scheduled_sanitizer_test.py PlanarSBA $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS

		# GCOV/LCOV coverage analysis of self_test suite
		coverage-test-on-main-scheduled:
		extends:
		- .test-job-linux-needs-testv-dir
		- .rules-main-scheduled
		tags:
		- coverage-test
		stage: test
		rules:
		# only run in scheduled pipeline that passes this env vars
		- if: $SANITIZER_TEST_IN_FMT && $SANITIZER_TEST_OUT_FMTS && $SANITIZER_TEST_TESTS
		- if: $COVERAGE_TEST
		script:
		- *print-common-info
		- echo "Running scheduled sanitizer tests $SANITIZER_TEST_TESTS for input format $SANITIZER_TEST_IN_FMT and output format(s) $SANITIZER_TEST_OUT_FMTS"

		- python3 ci/run_scheduled_sanitizer_test.py $SANITIZER_TEST_IN_FMT $SANITIZER_TEST_OUT_FMTS --tests $SANITIZER_TEST_TESTS
		- make GCOV=1 -j
		- ./scripts/self_test.py --create -t 1
		- ./scripts/self_test.py --create -t 1 scripts/config/self_test_evs.prm
		- ./scripts/ivas_pytests/self_test_b.py --create_only --numprocesses 1 --encref IVAS_cod --decref IVAS_dec --encdut IVAS_cod --decdut IVAS_dec
		- lcov -c -d obj -o coverage.info
		- genhtml coverage.info -o coverage
		artifacts:
		name: "sanitizer-test-results-and-error_pattern-$SANITIZER_TEST_IN_FMT-in-$SANITIZER_TEST_OUT_FMTS-out"
		name: "main-coverage-sha-$CI_COMMIT_SHORT_SHA"
		when: always
		paths:
		- ep_015.g192
		- "./*/logs"

		- coverage.info
		- coverage

		# ---------------------------------------------------------------
		# Other jobs

ci/run_scheduled_sanitizer_test.py

+14 −3

Original line number	Diff line number	Diff line
		@@ -32,16 +32,27 @@ def main(args):


		def get_modes(in_format: str) -> list:
		cmd = [SCRIPT_DIR.joinpath("runIvasCodec.py"), "-l"]

		cmd = [
		SCRIPT_DIR.joinpath("runIvasCodec.py"),
		"-C",
		"MC" if in_format in MC_MODES else in_format,
		"-l"
		]
		list_process = subprocess.run(cmd, capture_output=True)

		output = list_process.stdout.decode("utf8")

		# correction for multichannel modes to avoid selecting some mono modes...
		if in_format in MC_MODES:
		in_format = "MC_" + in_format
		in_format = "MC_" + in_format + "_b"

		mode_list = [m for m in output.splitlines() if in_format in m]
		if "SBA" in in_format:
		# rate switching not implemented yet
		mode_list = [m for m in mode_list if not "_rs" in m]

		return [m for m in output.splitlines() if in_format in m]
		return mode_list


		def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True):

lib_com/ivas_prot.h

+23 −9

Original line number	Diff line number	Diff line
		@@ -4243,16 +4243,30 @@ int16_t ivas_map_num_decd_r_to_idx( const int16_t num_quant_points_decd_r );

		/* Quantization utilities */
		void ivas_quantise_real_values(
		#ifdef QUANTISE_REAL_FCN_CLEAN_UP
		const float *values,
		#else
		float **values,
		#endif
		const int16_t q_levels,
		const float min_value,
		const float max_value,
		#ifdef QUANTISE_REAL_FCN_CLEAN_UP
		int16_t *index,
		float *quant,
		#else
		int16_t **index,
		float **quant,
		#endif
		#ifdef QUANTISE_REAL_FCN_CLEAN_UP
		const int16_t dim
		#else
		const int16_t dim1,
		const int16_t dim2
		#endif
		);


		void ivas_spar_get_uniform_quant_strat(
		ivas_spar_md_com_cfg *pSpar_md_com_cfg,
		const int16_t table_idx

lib_com/ivas_spar_com_quant_util.c

+49 −10

Original line number	Diff line number	Diff line
		@@ -41,7 +41,7 @@
		#include "ivas_rom_com.h"
		#include <assert.h>
		#include "wmops.h"

		#ifndef QUANTISE_REAL_FCN_CLEAN_UP
		/-----------------------------------------------------------------------------------------
		* Function ivas_limit_values()
		*
		@@ -56,7 +56,6 @@ static void ivas_limit_values(
		const int16_t dim2 )
		{
		int16_t i, j;

		for ( i = 0; i < dim1; i++ )
		{
		for ( j = 0; j < dim2; j++ )
		@@ -64,48 +63,73 @@ static void ivas_limit_values(
		ppValues[i][j] = max( min_value, min( ppValues[i][j], max_value ) );
		}
		}

		return;
		}

		#endif

		/-----------------------------------------------------------------------------------------
		* Function ivas_quantise_real_values()
		*
		* Quantize real values
		-----------------------------------------------------------------------------------------/

		void ivas_quantise_real_values(
		#ifdef QUANTISE_REAL_FCN_CLEAN_UP
		const float *values,
		#else
		float **values,
		#endif
		const int16_t q_levels,
		const float min_value,
		const float max_value,
		#ifdef QUANTISE_REAL_FCN_CLEAN_UP
		int16_t *index,
		float *quant,
		#else
		int16_t **index,
		float **quant,
		#endif
		#ifdef QUANTISE_REAL_FCN_CLEAN_UP
		const int16_t dim
		#else
		const int16_t dim1,
		const int16_t dim2 )
		const int16_t dim2
		#endif
		)
		{
		#ifndef QUANTISE_REAL_FCN_CLEAN_UP
		int16_t i, j;
		#else
		int16_t i;
		#endif
		float q_step, one_by_q_step;

		if ( q_levels == 1 )
		{
		#ifndef QUANTISE_REAL_FCN_CLEAN_UP
		for ( i = 0; i < dim1; i++ )
		{
		for ( j = 0; j < dim2; j++ )
		{

		quant[i][j] = 0;
		index[i][j] = 0;
		}
		}
		#else
		for ( i = 0; i < dim; i++ )
		{
		quant[i] = 0;
		index[i] = 0;
		}
		#endif
		}
		else if ( q_levels && max_value != min_value )
		{
		#ifndef QUANTISE_REAL_FCN_CLEAN_UP
		ivas_limit_values( values, min_value, max_value, dim1, dim2 );

		#endif
		q_step = ( max_value - min_value ) / ( q_levels - 1 );
		one_by_q_step = ( q_levels - 1 ) / ( max_value - min_value );

		#ifndef QUANTISE_REAL_FCN_CLEAN_UP
		for ( i = 0; i < dim1; i++ )
		{
		for ( j = 0; j < dim2; j++ )
		@@ -114,9 +138,19 @@ void ivas_quantise_real_values(
		quant[i][j] = index[i][j] * q_step;
		}
		}
		#else
		float val;
		for ( i = 0; i < dim; i++ )
		{
		val = max( min_value, min( values[i], max_value ) );
		index[i] = (int16_t) round( one_by_q_step * val );
		quant[i] = index[i] * q_step;
		}
		#endif
		}
		else
		{
		#ifndef QUANTISE_REAL_FCN_CLEAN_UP
		for ( i = 0; i < dim1; i++ )
		{
		for ( j = 0; j < dim2; j++ )
		@@ -124,8 +158,13 @@ void ivas_quantise_real_values(
		quant[i][j] = values[i][j];
		}
		}
		#else
		for ( i = 0; i < dim; i++ )
		{
		quant[i] = values[i];
		}
		#endif
		}

		return;
		}

lib_com/options.h

+1 −0

Original line number	Diff line number	Diff line
		@@ -150,6 +150,7 @@
		#define SBA_INTERN_CONFIG_FIX_HOA2 /* Issue 99 : Fix for incorrect internal_config when output format is HOA2 or FOA*/
		#define FIX_I98_HANDLES_TO_NULL /* Issue 98: do the setting of all handles to NULL in one place */

		#define QUANTISE_REAL_FCN_CLEAN_UP /Clean up the ivas_quantise_real_values() function/

		/* ################## End DEVELOPMENT switches ######################### */
		/* clang-format on */