Merge branch 'main' into 31-develop-performance-measurement-build (541d2d41) · Commits · IVAS Codec Public Collaboration / IVAS Codec

.gitlab-ci.yml

+2 −2

Original line number	Diff line number	Diff line
		@@ -724,8 +724,8 @@ codec-comparison-on-main-push:
		when: always
		paths:
		- ep_015.g192
		# second wildcard is necessary to get encoder and no-PLC run logs
		- "CLANG/logs"
		- ./LOGS_PLC
		- ./LOGS_noPLC

		### --- sanitizer schedule A ---

Workspace_msvc/renderer.vcxproj

+4 −0

Original line number	Diff line number	Diff line
		@@ -223,6 +223,10 @@
		<Project>{54509728-928B-44D9-A118-A6F92F08B34F}</Project>
		<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
		</ProjectReference>
		<ProjectReference Include="lib_dec.vcxproj">
		<Project>{E822DDAF-0F5F-4CD0-A694-38AE69DE74D3}</Project>
		<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
		</ProjectReference>
		<ProjectReference Include="lib_util.vcxproj">
		<Project>{2FA8F384-0775-F3B7-F8C3-85209222FC70}</Project>
		<ReferenceOutputAssembly>false</ReferenceOutputAssembly>

ci/collect_artifacts.py

0 → 100755

+103 −0

Original line number	Diff line number	Diff line
		#! /usr/bin/env python3
		import pathlib
		import argparse
		import re


		TEST_TYPES = ["sanitizers"]


		def main(args):

		test = args.test
		file = args.console_out_file
		if test == "sanitizers":
		collect_for_sanitizer_test(file)


		def find_failed_files_for_sanitizer_test(
		console_log: list, subfolder: str, which="LOGS"
		) -> dict():

		assert which in ["LOGS", "FILE_BASENAMES"]

		pattern_line = "(Encoding\|Decoding) failed .for \/.(CLANG.\|VALGRIND)\/(.*)"
		pattern_file = "(._b[0-9]_._rs\|._b[0-9]_._cbr).*"

		files_found = dict()
		for line in console_log:
		m_line = re.match(pattern_line, line)

		if m_line is not None:
		_, test, filename = m_line.groups()
		filename = pathlib.Path(filename).name
		m_file = re.match(pattern_file, filename)
		if m_file is None:
		print(f"Unexpected: no match on {filename} with {pattern_file} - skip")
		continue
		filename_start = m_file.groups()[0]

		if which == "LOGS":
		folder = pathlib.Path(f"{test}/{subfolder}/")
		files = [
		f for f in folder.iterdir() if f.name.startswith(filename_start)
		]
		elif which == "FILE_BASENAMES":
		files = [filename_start]
		if test in files_found:
		files_found[test].extend(files)
		else:
		files_found[test] = files

		return files_found


		def collect_for_sanitizer_test(file):

		with open(file) as f:
		console_log = f.readlines()

		files_to_archive_noPLC = find_failed_files_for_sanitizer_test(
		console_log, "logs_noPLC"
		)
		files_to_archive = find_failed_files_for_sanitizer_test(console_log, "logs")

		log_folder = pathlib.Path("./LOGS_PLC")
		log_folder.mkdir()
		for test in files_to_archive.keys():
		log_folder.joinpath(test).mkdir()
		for test, files in files_to_archive.items():
		folder = log_folder.joinpath(test)
		for p in files:
		source = pathlib.Path(p)
		target = folder.joinpath(source.name)
		source.rename(target)

		log_folder_noPLC = pathlib.Path("./LOGS_noPLC")
		log_folder_noPLC.mkdir()
		for test in files_to_archive_noPLC.keys():
		log_folder_noPLC.joinpath(test).mkdir()
		for test, files in files_to_archive_noPLC.items():
		folder = log_folder_noPLC.joinpath(test)
		for p in files:
		source = pathlib.Path(p)
		target = folder.joinpath(source.name)
		source.rename(target)


		if __name__ == "__main__":
		parser = argparse.ArgumentParser()
		parser.add_argument(
		"test",
		type=str,
		choices=TEST_TYPES,
		help="for which test should artifacts be collected?",
		)
		parser.add_argument(
		"console_out_file",
		type=str,
		help="file with stdout from IvasBuildAndRunChecks.py",
		)
		args = parser.parse_args()

		main(args)

ci/run_scheduled_sanitizer_test.py

100644 → 100755

+58 −15

Original line number	Diff line number	Diff line
		@@ -34,6 +34,13 @@ import sys
		import subprocess
		import pathlib

		CI_SCRIPT_DIR = "./ci"
		sys.path.append(CI_SCRIPT_DIR)
		from collect_artifacts import (
		find_failed_files_for_sanitizer_test,
		collect_for_sanitizer_test,
		)


		DURATION = "120"
		CFG = "ci_linux_ltv.json"
		@@ -45,6 +52,8 @@ MC_MODES = ["5_1", "5_1_2", "5_1_4", "7_1", "7_1_4"]

		SCRIPT_DIR = pathlib.Path("./scripts").resolve()

		CONSOLE_OUT_FILE = "output_san.txt"


		def main(args):
		in_format = args.in_format
		@@ -57,6 +66,8 @@ def main(args):
		modes = get_modes(in_format)
		returncode = run_check(modes, out_formats, tests, run_fec=run_fec)

		collect_for_sanitizer_test(CONSOLE_OUT_FILE)

		sys.exit(returncode)


		@@ -66,7 +77,7 @@ def get_modes(in_format: str) -> list:
		SCRIPT_DIR.joinpath("runIvasCodec.py"),
		"-C",
		"MC" if in_format in MC_MODES else in_format,
		"-l"
		"-l",
		]
		list_process = subprocess.run(cmd, capture_output=True)

		@@ -78,6 +89,11 @@ def get_modes(in_format: str) -> list:
		in_format = "MC_" + in_format + "_b"
		mode_list = [m for m in mode_list if in_format in m]

		# TODO: remove once #185 is fixed
		# temporarily skip 24.4kbps SBA bitrate
		if in_format == "SBA":
		mode_list = [m for m in mode_list if not "b24_4" in m]

		return mode_list


		@@ -98,21 +114,43 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True)
		*out_formats,
		]

		print("======== Script command line WITHOUT plc: ========\n{}".format(" ".join(cmd_no_fec)))
		print(
		"======== Script command line WITHOUT plc: ========\n{}".format(
		" ".join(cmd_no_fec)
		)
		)

		proc = subprocess.Popen(cmd_no_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
		with open(CONSOLE_OUT_FILE, "a") as f:
		proc = subprocess.Popen(
		cmd_no_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE
		)
		for c in iter(lambda: proc.stdout.read(1), b""):
		sys.stdout.buffer.write(c)
		f.write(c.decode("utf8"))
		proc.wait()

		if proc.returncode not in [0, 101]:
		raise IvasBuildAndRunFailed("Failed at first run (no PLC)")

		returncode_no_fec = proc.returncode
		print("returncode_no_fec:", returncode_no_fec)
		if returncode_no_fec not in [0, 101]:
		raise IvasBuildAndRunFailed("Failed at first run (no PLC)")

		if not run_fec:
		return returncode_no_fec

		# delete bitstream files for all failed modes to prevent follow-up errors in decoder-only run
		with open(CONSOLE_OUT_FILE) as f:
		console_log = f.readlines()
		failed_files = find_failed_files_for_sanitizer_test(
		console_log, "logs", "FILE_BASENAMES"
		)
		for t in failed_files.keys():
		bs_folder = pathlib.Path(f"{t}/enc")
		file_starts = failed_files[t]
		for f in bs_folder.iterdir():
		for fs in file_starts:
		if f.name.startswith(fs):
		f.unlink()

		### second run: decoder only with disturbed bitstream

		# generate error pattern
		@@ -135,7 +173,11 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True)
		path.mkdir()

		cmd_fec = cmd_no_fec + ["--decoder_only", "-f", EP_FILE]
		print("======== Script command line WITH plc: ========\n{}".format(" ".join(cmd_no_fec)))
		print(
		"======== Script command line WITH plc: ========\n{}".format(
		" ".join(cmd_no_fec)
		)
		)

		proc = subprocess.Popen(cmd_fec, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
		for c in iter(lambda: proc.stdout.read(1), b""):
		@@ -143,6 +185,7 @@ def run_check(modes: list, out_formats: list, tests: list, run_fec: bool = True)
		proc.wait()

		returncode_fec = proc.returncode
		print("returncode_fec:", returncode_fec)

		if returncode_fec not in [0, 101]:
		raise IvasBuildAndRunFailed("failed at second run (PLC)")

lib_com/options.h

100644 → 100755

+2 −1

Original line number	Diff line number	Diff line
		@@ -153,7 +153,8 @@
		#define SBA_BR_SWITCHING /* Issue 114: Changes for sba bit rate switching*/
		#define FIX_AGC_WINFUNC_MEMORY /* Issue 62: lower agc_com.winFunc memory consumption */
		#define REMOVE_SID_HARM_LEFTOVERS /* Issue 192: remove leftovers from the SID bitrate harmonization */

		#define FIX_MCT_UNINIT_MEM /* Issue 166: Reading of uninitialized memory in TCX range coder */
		#define FIX_IGF_NOISE_REPETITION /* Issue 182: fix repetition of same noise in IGF */


		/* ################## End DEVELOPMENT switches ######################### */