Merge branch '4-migrate-to-pytest-framework' of... (577993f4) · Commits · IVAS Codec Public Collaboration / IVAS Codec

.gitlab-ci.yml

+248 −14

Original line number	Diff line number	Diff line
		@@ -2,6 +2,10 @@ variables:
		TESTV_DIR: "/usr/local/testv"
		BUILD_OUTPUT: "build_output.txt"
		EVS_BE_TEST_DIR: "/usr/local/be_2_evs_test"
		SANITIZER_TESTS: "CLANG1 CLANG2"
		OUT_FORMATS_CHANNEL_BASED: "stereo mono 5_1 5_1_2 5_1_4 7_1 7_1_4"
		OUT_FORMATS_SCENE_BASED: "FOA HOA2 HOA3"
		OUT_FORMATS_BINAURAL: "BINAURAL BINAURAL_ROOM"
		EXIT_CODE_NON_BE: 123
		EXIT_CODE_FAIL: 1

		@@ -16,7 +20,6 @@ workflow:
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main
		- if: $CI_PIPELINE_SOURCE == 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Scheduled in main


		stages:
		- maintenance
		- build
		@@ -54,6 +57,8 @@ stages:
		rules:
		- if: $MIRROR_ACCESS_TOKEN # Don't run in the mirror update pipeline (only then MIRROR_ACCESS_TOKEN is defined)
		when: never
		- if: $CI_PIPELINE_SOURCE == 'schedule' # Don't run in any scheduled pipelines by default (use schedule templates below to enable again for certain conditions)
		when: never
		- when: on_success

		.rules-merge-request:
		@@ -219,6 +224,12 @@ msan-on-merge-request-linux:
		- python3 scripts/self_test.py --create \| tee test_output.txt
		- run_errors=$(cat test_output.txt \| grep -ic "run errors") \|\| true
		- if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py with Clang memory-sanitizer"; exit 1; fi
		artifacts:
		name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"
		paths:
		- scripts/ref/logs/
		- test_output.txt
		expose_as: 'Msan selftest results'


		# code selftest testvectors with address-sanitizer binaries
		@@ -235,6 +246,12 @@ asan-on-merge-request-linux:
		- python3 scripts/self_test.py --create \| tee test_output.txt
		- run_errors=$(cat test_output.txt \| grep -ic "run errors") \|\| true
		- if [ $run_errors != 0 ] ; then echo "Run errors in self_test.py with Clang address-sanitizer"; exit 1; fi
		artifacts:
		name: "mr-$CI_MERGE_REQUEST_IID--sha-$CI_COMMIT_SHORT_SHA--stage-$CI_JOB_STAGE--results"
		paths:
		- scripts/ref/logs/
		- test_output.txt
		expose_as: 'Asan selftest results'


		# compare bit exactness between target and source branch
		@@ -283,8 +300,13 @@ pytest-on-merge-request:
		### re-checkout the commit from the source branch to have up-to-date test scripts and test vectors (and actually everything)
		- git checkout $source_branch_commit_sha

		# helper variable - "\|\| true" to prevent failures from grep not finding anything
		# some helper variables - "\|\| true" to prevent failures from grep not finding anything
		- evs_non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE \| grep -c --ignore-case "\[evs[ -]non[ -]*be\]") \|\| true
		- non_be_flag=$(echo $CI_MERGE_REQUEST_TITLE \| grep -c --ignore-case "\[non[ -]*be\]") \|\| true
		- expected_nonbe_1=0
		- expected_nonbe_2=0
		- fail_1=0
		- fail_2=0

		### prepare pytest
		# create short test vectors
		@@ -295,14 +317,33 @@ pytest-on-merge-request:
		# create references
		- python3 -m pytest tests -n auto --update_ref 1 -m create_ref
		- python3 -m pytest tests -n auto --update_ref 1 -m create_ref_part2
		- python3 -m pytest tests/test_param_file.py -n auto --update_ref 1 -m create_ref --param_file scripts/config/self_test_evs.prm

		### run pytest
		- exit_code=0
		- python3 -m pytest tests -n auto --junit-xml=report-junit.xml \|\| exit_code=$?
		- if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures and non-BE flag not present"; exit $EXIT_CODE_FAIL; fi
		- zero_errors=$(cat report-junit.xml \| grep -c 'errors="0"') \|\| true
		- if [ $exit_code -eq 1 ] && [ $zero_errors == 1 ]; then echo "pytest run had failures, but no errors and non-BE flag present"; exit $EXIT_CODE_NON_BE; fi
		- if [ $exit_code -ne 0 ]; then echo "pytest run had errors"; exit $EXIT_CODE_FAIL; fi;

		- if [ $zero_errors != 1 ]; then echo "Run errors in pytest"; fail_1=1; fi

		- if [ $exit_code -eq 1 ] && [ $non_be_flag == 0 ]; then echo "pytest run had failures without non-BE tag encountered"; fail_1=1; fi
		- if [ $exit_code -eq 1 ] && [ $non_be_flag != 0 ]; then echo "pytest run had failures with non-BE tag encountered"; expected_nonbe_1=1; fi

		### run pytest for EVS cases
		- exit_code=0
		- python3 -m pytest tests/test_param_file.py -n auto --param_file scripts/config/self_test_evs.prm --junit-xml=report-junit-evs.xml \|\| exit_code=$?
		- zero_errors=$(cat report-junit-evs.xml \| grep -c 'errors="0"') \|\| true

		- if [ $zero_errors != 1 ]; then echo "Run errors in pytest for EVS"; fail_2=1; fi

		- if [ $exit_code -eq 1 ] && [ $evs_non_be_flag == 0 ]; then echo "Non-bitexact EVS cases without EVS-non-BE tag encountered"; fail_2=1; fi
		- if [ $exit_code -eq 1 ] && [ $evs_non_be_flag != 0 ]; then echo "Non-bitexact EVS cases with EVS-non-BE tag encountered"; expected_nonbe_2=1; fi

		# Check results from both tests
		- if [ $fail_1 -eq 1 ] \|\| [ $fail_2 -eq 1 ]; then exit $EXIT_CODE_FAIL; fi
		- if [ $expected_nonbe_1 -eq 1 ] \|\| [ $expected_nonbe_2 -eq 1 ]; then exit $EXIT_CODE_NON_BE; fi
		- exit 0

		allow_failure:
		exit_codes:
		- 123
		@@ -311,9 +352,12 @@ pytest-on-merge-request:
		when: always
		paths:
		- report-junit.xml
		expose_as: 'Self test results'
		- report-junit-evs.xml
		expose_as: 'pytest results'
		reports:
		junit: report-junit.xml
		junit:
		- report-junit.xml
		- report-junit-evs.xml


		# ---------------------------------------------------------------
		@@ -329,6 +373,7 @@ be-2-evs-linux:
		- be-2-evs-temp
		stage: test
		needs: [ "build-codec-linux-cmake" ]
		timeout: "20 minutes" # To be revisited
		script:
		- *print-common-info

		@@ -422,17 +467,206 @@ codec-comparison-on-main-push:
		junit: report-junit.xml


		sanitizer-test-on-main-scheduled:
		extends: .test-job-linux-needs-testv-dir
		# ---------------------------------------------------------------
		# Scheduled jobs on main
		# ---------------------------------------------------------------
		.sanitizer-test-template:
		extends:
		- .test-job-linux-needs-testv-dir
		stage: test
		tags:
		- sanitizer_test_main

		sanitizer-test-mono:
		extends: .sanitizer-test-template
		rules:
		# only run in scheduled pipeline that passes this env var
		- if: $SANITIZER_TEST_IN_FMT
		- if: $IS_SANITIZER_TEST_RUN
		script:
		- *print-common-info
		- echo "Running scheduled sanitizer"
		# - python3 ci/run_scheduled_sanitizer_test.py $SANITIZER_TEST_IN_FMT $SANITIZER_TEST_OUT_FMTS
		- python3 ci/run_scheduled_sanitizer_test.py mono mono --tests $SANITIZER_TESTS

		sanitizer-test-stereo:
		extends: .sanitizer-test-template
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		when: delayed
		start_in: 20 minutes
		script:
		- python3 ci/run_scheduled_sanitizer_test.py stereo $OUT_FORMATS_CHANNEL_BASED --tests $SANITIZER_TESTS

		sanitizer-test-stereodmxevs:
		extends: .sanitizer-test-template
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		when: delayed
		start_in: 40 minutes
		script:
		- python3 ci/run_scheduled_sanitizer_test.py StereoDmxEvs mono --tests $SANITIZER_TESTS

		sanitizer-test-ism1:
		extends: .sanitizer-test-template
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		when: delayed
		start_in: 1 hour
		script:
		- python3 ci/run_scheduled_sanitizer_test.py ISM1 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS

		sanitizer-test-ism2:
		extends: .sanitizer-test-template
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		when: delayed
		start_in: 1 hour 30 minutes
		script:
		- python3 ci/run_scheduled_sanitizer_test.py ISM2 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS

		sanitizer-test-ism3:
		extends: .sanitizer-test-template
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		when: delayed
		start_in: 2 hours
		script:
		- python3 ci/run_scheduled_sanitizer_test.py ISM3 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS

		sanitizer-test-ism4:
		extends: .sanitizer-test-template
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		when: delayed
		start_in: 2 hours 30 minutes
		script:
		- python3 ci/run_scheduled_sanitizer_test.py ISM4 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS

		sanitizer-test-mc-5_1:
		extends:
		- .test-job-linux-needs-testv-dir
		stage: test
		tags:
		- sanitizer_test_main
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		when: delayed
		start_in: 3 hours
		script:
		- python3 ci/run_scheduled_sanitizer_test.py 5_1 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS

		sanitizer-test-mc-5_1_2:
		extends:
		- .test-job-linux-needs-testv-dir
		stage: test
		tags:
		- sanitizer_test_main
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		when: delayed
		start_in: 4 hours
		script:
		- python3 ci/run_scheduled_sanitizer_test.py 5_1_2 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS

		sanitizer-test-mc-5_1_4:
		extends:
		- .test-job-linux-needs-testv-dir
		stage: test
		tags:
		- sanitizer_test_main
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		when: delayed
		start_in: 5 hours
		script:
		- python3 ci/run_scheduled_sanitizer_test.py 5_1_4 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS

		sanitizer-test-mc-7_1:
		extends:
		- .test-job-linux-needs-testv-dir
		stage: test
		tags:
		- sanitizer_test_main
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		when: delayed
		start_in: 6 hours
		script:
		- python3 ci/run_scheduled_sanitizer_test.py 7_1 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS

		sanitizer-test-mc-7_1_4:
		extends:
		- .test-job-linux-needs-testv-dir
		stage: test
		tags:
		- sanitizer_test_main
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		when: delayed
		start_in: 7 hours
		script:
		- python3 ci/run_scheduled_sanitizer_test.py 7_1_4 $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS

		sanitizer-test-masa:
		extends:
		- .test-job-linux-needs-testv-dir
		stage: test
		tags:
		- sanitizer_test_main
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		when: delayed
		start_in: 8 hours
		script:
		- python3 ci/run_scheduled_sanitizer_test.py MASA $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL EXT --tests $SANITIZER_TESTS

		sanitizer-test-sba:
		extends:
		- .test-job-linux-needs-testv-dir
		stage: test
		tags:
		- sanitizer_test_main
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		when: delayed
		start_in: 9 hours
		script:
		- python3 ci/run_scheduled_sanitizer_test.py SBA $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS

		sanitizer-test-planarsba:
		extends:
		- .test-job-linux-needs-testv-dir
		stage: test
		tags:
		- sanitizer_test_main
		rules:
		- if: $IS_SANITIZER_TEST_RUN
		when: delayed
		start_in: 10 hours
		script:
		- python3 ci/run_scheduled_sanitizer_test.py PlanarSBA $OUT_FORMATS_CHANNEL_BASED $OUT_FORMATS_SCENE_BASED $OUT_FORMATS_BINAURAL --tests $SANITIZER_TESTS

		# GCOV/LCOV coverage analysis of self_test suite
		coverage-test-on-main-scheduled:
		extends:
		- .test-job-linux-needs-testv-dir
		- .rules-main-scheduled
		tags:
		- coverage-test
		stage: test
		rules:
		# only run in scheduled pipeline that passes this env vars
		- if: $COVERAGE_TEST
		script:
		- *print-common-info
		- make GCOV=1 -j
		- ./scripts/self_test.py --create -t 1
		- ./scripts/self_test.py --create -t 1 scripts/config/self_test_evs.prm
		- ./scripts/ivas_pytests/self_test_b.py --create_only --numprocesses 1 --encref IVAS_cod --decref IVAS_dec --encdut IVAS_cod --decdut IVAS_dec
		- lcov -c -d obj -o coverage.info
		- genhtml coverage.info -o coverage
		artifacts:
		name: "main-coverage-sha-$CI_COMMIT_SHORT_SHA"
		when: always
		paths:
		- coverage.info
		- coverage

		# ---------------------------------------------------------------
		# Other jobs

ci/check_for_warnings.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -3,7 +3,7 @@ import argparse
		import sys


		SEARCH_FOR = "warning:"
		SEARCH_FOR = "warning"
		RETURN_FOUND = 123

ci/run_scheduled_sanitizer_test.py

0 → 100644

+130 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env python3

		import argparse
		import sys
		import subprocess
		import pathlib


		DURATION = "120"
		CFG = "ci_linux.json"
		SUPPORTED_TESTS = ["CLANG1", "CLANG2", "CLANG3", "VALGRIND"]
		EP_FILE = "ep_015.g192"
		GENPATT_CMD = f"gen-patt -tailstat -fer -g192 -gamma 0 -rate 0.15 -tol 0.001 -reset -n {int(DURATION) * 50} {EP_FILE}"
		EIDXOR_CMD = "eid-xor -vbr -fer {bitstream} {ep_file} {out_file}"
		MC_MODES = ["5_1", "5_1_2", "5_1_4", "7_1", "7_1_4"]

		SCRIPT_DIR = pathlib.Path("./scripts").resolve()


		def main(args):
		in_format = args.in_format
		out_formats = args.out_formats
		tests = args.tests
		run_fec = not args.skip_fec

		assert all([t in SUPPORTED_TESTS for t in tests])

		modes = get_modes(in_format)
		returncode = run_check(modes, out_formats, tests, run_fec=run_fec)

		sys.exit(returncode)


		def get_modes(in_format: str) -> list:

		cmd = [
		SCRIPT_DIR.joinpath("runIvasCodec.py"),
		"-C",
		"MC" if in_format in MC_MODES else in_format,
		"-l"
		]
		list_process = subprocess.run(cmd, capture_output=True)

		output = list_process.stdout.decode("utf8")

		# correction for multichannel modes to avoid selecting some mono modes...
		if in_format in MC_MODES:
		in_format = "MC_" + in_format + "_b"

		mode_list = [m for m in output.splitlines() if in_format in m]
		if "SBA" in in_format:
		# rate switching not implemented yet
		mode_list = [m for m in mode_list if not "_rs" in m]

		return mode_list


		def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True):

		### always run encoder and decoder with no frameloss
		cmd_no_fec = [
		str(SCRIPT_DIR.joinpath("IvasBuildAndRunChecks.py")),
		"-U",
		DURATION,
		"-p",
		CFG,
		"--checks",
		*tests,
		"-m",
		*modes,
		"--oc",
		*out_formats,
		]

		print("======== Script command line WITHOUT plc: ========\n{}".format(" ".join(cmd_no_fec)))

		proc = subprocess.Popen(cmd_no_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
		for c in iter(lambda: proc.stdout.read(1), b""):
		sys.stdout.buffer.write(c)
		proc.wait()

		if proc.returncode not in [0, 101]:
		raise IvasBuildAndRunFailed("Failed at first run (no PLC)")

		returncode_no_fec = proc.returncode

		if not run_fec:
		return returncode_no_fec

		### second run: decoder only with disturbed bitstream

		# generate error pattern
		subprocess.call(GENPATT_CMD.split())

		# cleanup to avoid script errors
		# we want "logs" and "dec" subfolders to be empty -> delete and recreate them
		cleanup_folders = ["logs", "dec"]
		for t in tests:
		for fol in cleanup_folders:
		for fi in pathlib.Path(t).joinpath(fol).iterdir():
		fi.unlink()

		cmd_fec = cmd_no_fec + ["--decoder_only", "-f", EP_FILE]
		print("======== Script command line WITH plc: ========\n{}".format(" ".join(cmd_no_fec)))

		proc = subprocess.Popen(cmd_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
		for c in iter(lambda: proc.stdout.read(1), b""):
		sys.stdout.buffer.write(c)
		proc.wait()

		returncode_fec = proc.returncode

		if returncode_fec not in [0, 101]:
		raise IvasBuildAndRunFailed("failed at second run (PLC)")

		return 101 if 101 in [returncode_no_fec, returncode_fec] else 0


		class IvasBuildAndRunFailed(Exception):
		pass


		if __name__ == "__main__":
		parser = argparse.ArgumentParser()
		parser.add_argument("in_format", type=str)
		parser.add_argument("out_formats", type=str, nargs="+")
		parser.add_argument("--tests", type=str, nargs="+", default=["CLANG1", "CLANG2"])
		parser.add_argument("--skip_fec", action="store_true")

		sys.exit(main(parser.parse_args()))

lib_com/ivas_cnst.h

+1 −1

Original line number	Diff line number	Diff line
		@@ -1135,7 +1135,7 @@ enum
		#define MASA_STEREO_MIN_BITRATE IVAS_24k4

		#define MASA_BIT_REDUCT_PARAM 10

		#define MASA_MAXIMUM_TWO_DIR_BANDS 18
		typedef enum
		{
		MASA_STEREO_NOT_DEFINED,

lib_com/ivas_prot.h

+29 −9

Original line number	Diff line number	Diff line
		@@ -238,6 +238,12 @@ uint32_t ivas_syn_output(
		int16_t synth_out / o : integer 16 bits synthesis signal */
		);

		#ifdef FIX_I98_HANDLES_TO_NULL
		void ivas_initialize_handles_enc(
		Encoder_Struct st_ivas / i/o: IVAS encoder structure */
		);
		#endif

		ivas_error ivas_init_encoder(
		Encoder_Struct st_ivas, / i/o: IVAS encoder structure */
		Indice ind_list[][MAX_NUM_INDICES], /* i : indices list */
		@@ -4237,16 +4243,30 @@ int16_t ivas_map_num_decd_r_to_idx( const int16_t num_quant_points_decd_r );

		/* Quantization utilities */
		void ivas_quantise_real_values(
		#ifdef QUANTISE_REAL_FCN_CLEAN_UP
		const float *values,
		#else
		float **values,
		#endif
		const int16_t q_levels,
		const float min_value,
		const float max_value,
		#ifdef QUANTISE_REAL_FCN_CLEAN_UP
		int16_t *index,
		float *quant,
		#else
		int16_t **index,
		float **quant,
		#endif
		#ifdef QUANTISE_REAL_FCN_CLEAN_UP
		const int16_t dim
		#else
		const int16_t dim1,
		const int16_t dim2
		#endif
		);


		void ivas_spar_get_uniform_quant_strat(
		ivas_spar_md_com_cfg *pSpar_md_com_cfg,
		const int16_t table_idx