Merge from main (f4f3e564) · Commits · IVAS Codec Public Collaboration / IVAS Codec

.gitlab-ci.yml

+4 −3

Original line number	Diff line number	Diff line
		@@ -20,6 +20,7 @@ workflow:
		- if: $CI_PIPELINE_SOURCE == 'merge_request_event' # Runs for merge requests
		- if: $CI_PIPELINE_SOURCE == 'push' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Pushes to main
		- if: $CI_PIPELINE_SOURCE == 'schedule' && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Scheduled in main
		- if: $CI_PIPELINE_SOURCE == "web"

		stages:
		- maintenance
		@@ -722,8 +723,8 @@ codec-comparison-on-main-push:
		when: always
		paths:
		- ep_015.g192
		# second wildcard is necessary to get encoder and no-PLC run logs
		- "CLANG/logs"
		- ./LOGS_PLC
		- ./LOGS_noPLC

		### --- sanitizer schedule A ---

		@@ -897,7 +898,7 @@ coverage-test-on-main-scheduled:
		- .test-job-linux-needs-testv-dir
		- .rules-main-scheduled
		tags:
		- coverage-test
		- ivas-linux
		stage: test
		rules:
		# only run in scheduled pipeline that passes this env vars

ci/collect_artifacts.py

0 → 100755

+103 −0

Original line number	Diff line number	Diff line
		#! /usr/bin/env python3
		import pathlib
		import argparse
		import re


		TEST_TYPES = ["sanitizers"]


		def main(args):

		test = args.test
		file = args.console_out_file
		if test == "sanitizers":
		collect_for_sanitizer_test(file)


		def find_failed_files_for_sanitizer_test(
		console_log: list, subfolder: str, which="LOGS"
		) -> dict():

		assert which in ["LOGS", "FILE_BASENAMES"]

		pattern_line = "(Encoding\|Decoding) failed .for \/.(CLANG.\|VALGRIND)\/(.*)"
		pattern_file = "(._b[0-9]_._rs\|._b[0-9]_._cbr).*"

		files_found = dict()
		for line in console_log:
		m_line = re.match(pattern_line, line)

		if m_line is not None:
		_, test, filename = m_line.groups()
		filename = pathlib.Path(filename).name
		m_file = re.match(pattern_file, filename)
		if m_file is None:
		print(f"Unexpected: no match on {filename} with {pattern_file} - skip")
		continue
		filename_start = m_file.groups()[0]

		if which == "LOGS":
		folder = pathlib.Path(f"{test}/{subfolder}/")
		files = [
		f for f in folder.iterdir() if f.name.startswith(filename_start)
		]
		elif which == "FILE_BASENAMES":
		files = [filename_start]
		if test in files_found:
		files_found[test].extend(files)
		else:
		files_found[test] = files

		return files_found


		def collect_for_sanitizer_test(file):

		with open(file) as f:
		console_log = f.readlines()

		files_to_archive_noPLC = find_failed_files_for_sanitizer_test(
		console_log, "logs_noPLC"
		)
		files_to_archive = find_failed_files_for_sanitizer_test(console_log, "logs")

		log_folder = pathlib.Path("./LOGS_PLC")
		log_folder.mkdir()
		for test in files_to_archive.keys():
		log_folder.joinpath(test).mkdir()
		for test, files in files_to_archive.items():
		folder = log_folder.joinpath(test)
		for p in files:
		source = pathlib.Path(p)
		target = folder.joinpath(source.name)
		source.rename(target)

		log_folder_noPLC = pathlib.Path("./LOGS_noPLC")
		log_folder_noPLC.mkdir()
		for test in files_to_archive_noPLC.keys():
		log_folder_noPLC.joinpath(test).mkdir()
		for test, files in files_to_archive_noPLC.items():
		folder = log_folder_noPLC.joinpath(test)
		for p in files:
		source = pathlib.Path(p)
		target = folder.joinpath(source.name)
		source.rename(target)


		if __name__ == "__main__":
		parser = argparse.ArgumentParser()
		parser.add_argument(
		"test",
		type=str,
		choices=TEST_TYPES,
		help="for which test should artifacts be collected?",
		)
		parser.add_argument(
		"console_out_file",
		type=str,
		help="file with stdout from IvasBuildAndRunChecks.py",
		)
		args = parser.parse_args()

		main(args)

ci/run_scheduled_sanitizer_test.py

100644 → 100755

+58 −15

Original line number	Diff line number	Diff line
		@@ -5,6 +5,13 @@ import sys
		import subprocess
		import pathlib

		CI_SCRIPT_DIR = "./ci"
		sys.path.append(CI_SCRIPT_DIR)
		from collect_artifacts import (
		find_failed_files_for_sanitizer_test,
		collect_for_sanitizer_test,
		)


		DURATION = "120"
		CFG = "ci_linux_ltv.json"
		@@ -16,6 +23,8 @@ MC_MODES = ["5_1", "5_1_2", "5_1_4", "7_1", "7_1_4"]

		SCRIPT_DIR = pathlib.Path("./scripts").resolve()

		CONSOLE_OUT_FILE = "output_san.txt"


		def main(args):
		in_format = args.in_format
		@@ -28,6 +37,8 @@ def main(args):
		modes = get_modes(in_format)
		returncode = run_check(modes, out_formats, tests, run_fec=run_fec)

		collect_for_sanitizer_test(CONSOLE_OUT_FILE)

		sys.exit(returncode)


		@@ -37,7 +48,7 @@ def get_modes(in_format: str) -> list:
		SCRIPT_DIR.joinpath("runIvasCodec.py"),
		"-C",
		"MC" if in_format in MC_MODES else in_format,
		"-l"
		"-l",
		]
		list_process = subprocess.run(cmd, capture_output=True)

		@@ -49,6 +60,11 @@ def get_modes(in_format: str) -> list:
		in_format = "MC_" + in_format + "_b"
		mode_list = [m for m in mode_list if in_format in m]

		# TODO: remove once #185 is fixed
		# temporarily skip 24.4kbps SBA bitrate
		if in_format == "SBA":
		mode_list = [m for m in mode_list if not "b24_4" in m]

		return mode_list


		@@ -69,21 +85,43 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True)
		*out_formats,
		]

		print("======== Script command line WITHOUT plc: ========\n{}".format(" ".join(cmd_no_fec)))
		print(
		"======== Script command line WITHOUT plc: ========\n{}".format(
		" ".join(cmd_no_fec)
		)
		)

		proc = subprocess.Popen(cmd_no_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
		with open(CONSOLE_OUT_FILE, "a") as f:
		proc = subprocess.Popen(
		cmd_no_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE
		)
		for c in iter(lambda: proc.stdout.read(1), b""):
		sys.stdout.buffer.write(c)
		f.write(c.decode("utf8"))
		proc.wait()

		if proc.returncode not in [0, 101]:
		raise IvasBuildAndRunFailed("Failed at first run (no PLC)")

		returncode_no_fec = proc.returncode
		print("returncode_no_fec:", returncode_no_fec)
		if returncode_no_fec not in [0, 101]:
		raise IvasBuildAndRunFailed("Failed at first run (no PLC)")

		if not run_fec:
		return returncode_no_fec

		# delete bitstream files for all failed modes to prevent follow-up errors in decoder-only run
		with open(CONSOLE_OUT_FILE) as f:
		console_log = f.readlines()
		failed_files = find_failed_files_for_sanitizer_test(
		console_log, "logs", "FILE_BASENAMES"
		)
		for t in failed_files.keys():
		bs_folder = pathlib.Path(f"{t}/enc")
		file_starts = failed_files[t]
		for f in bs_folder.iterdir():
		for fs in file_starts:
		if f.name.startswith(fs):
		f.unlink()

		### second run: decoder only with disturbed bitstream

		# generate error pattern
		@@ -106,7 +144,11 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True)
		path.mkdir()

		cmd_fec = cmd_no_fec + ["--decoder_only", "-f", EP_FILE]
		print("======== Script command line WITH plc: ========\n{}".format(" ".join(cmd_no_fec)))
		print(
		"======== Script command line WITH plc: ========\n{}".format(
		" ".join(cmd_no_fec)
		)
		)

		proc = subprocess.Popen(cmd_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
		for c in iter(lambda: proc.stdout.read(1), b""):
		@@ -114,6 +156,7 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True)
		proc.wait()

		returncode_fec = proc.returncode
		print("returncode_fec:", returncode_fec)

		if returncode_fec not in [0, 101]:
		raise IvasBuildAndRunFailed("failed at second run (PLC)")

lib_com/options.h

+2 −2

Original line number	Diff line number	Diff line
		@@ -153,8 +153,8 @@
		#define FIX_MCT_PLC_RECOVERY /* Issue 184: scale the old synthesis part correctly in the first good frame after lost frames in MCT modes - to be activated after previous switch is merged */
		#define SBA_BR_SWITCHING /* Issue 114: Changes for sba bit rate switching*/
		#define FIX_AGC_WINFUNC_MEMORY /* Issue 62: lower agc_com.winFunc memory consumption */
		#define REMOVE_SID_HARM_LEFTOVERS /* Issue 192: remove leftovers from the SID bitrate harmonization */

		#define FIX_MCT_UNINIT_MEM /* Issue 166: Reading of uninitialized memory in TCX range coder */
		#define FIX_IGF_NOISE_REPETITION /* Issue 182: fix repetition of same noise in IGF */


		/* ################## End DEVELOPMENT switches ######################### */

lib_dec/dec_tcx.c

100644 → 100755

+18 −0

Original line number	Diff line number	Diff line
		@@ -1309,8 +1309,25 @@ void decoder_tcx_noisefilling(
		IGFDecReplicateTCX10State( st->hIGFDec );
		}

		#ifdef FIX_IGF_NOISE_REPETITION
		if ( st->element_mode != EVS_MONO )
		{
		if ( bfi )
		{
		nf_seed = st->seed_tcx_plc;
		}
		else if ( nf_seed == 0 )
		{
		nf_seed = *st->hIGFDec->igfData.igfInfo.nfSeed;
		}
		}
		#endif

		if ( st->igf )
		{
		#ifdef FIX_IGF_NOISE_REPETITION
		st->hIGFDec->igfData.igfInfo.nfSeed = (int16_t) ( nf_seed 31821L + 13849L );
		#else
		if ( bfi && st->element_mode != EVS_MONO )
		{
		st->hIGFDec->igfData.igfInfo.nfSeed = (int16_t) ( st->seed_tcx_plc 31821L + 13849L );
		@@ -1319,6 +1336,7 @@ void decoder_tcx_noisefilling(
		{
		st->hIGFDec->igfData.igfInfo.nfSeed = (int16_t) ( nf_seed 31821L + 13849L );
		}
		#endif
		}

		return;