Merge branch 'mullerfa/python-improvements' into 'main' (014cf4f5) · Commits · IVAS Codec Public Collaboration / IVAS Codec

ci/smoke_test.sh

+24 −21

Original line number	Diff line number	Diff line
		@@ -28,6 +28,8 @@
		# accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and
		# the United Nations Convention on Contracts on the International Sales of Goods.

		set -euxo pipefail

		function usage {
		echo
		echo "Usage:"
		@@ -42,9 +44,9 @@ if [ ! -d "lib_com" ]; then
		exit 1
		fi

		if [ -z "$1" ] \|\| [ "$1" == "test" ]; then
		if [ -z "${1:-}" ] \|\| [ "${1:-}" == "test" ]; then
		BUILD=1
		elif [ "$1" == "coverage" ]; then
		elif [ "${1:-}" == "coverage" ]; then
		BUILD=0
		else
		usage
		@@ -76,37 +78,38 @@ if [ $BUILD -eq 1 ];then
		fi

		# prepare combined format test signals
		echo "\n======================= 0. preparing combined format test inputs =======================\n\n"
		echo -e "\n======================= 0. preparing combined format test inputs =======================\n\n"
		./scripts/prepare_combined_format_inputs.py

		# run all modes vanilla-fashion
		# treat ISM modes separately because passing the metadata files to MASA modes causes crashes
		ism_modes=$(./scripts/runIvasCodec.py -l \| grep ISM)
		non_ism_modes=$(./scripts/runIvasCodec.py -l \| grep -v ISM)
		echo "\n======================= 1. non-ism modes no FEC =======================\n\n"
		./scripts/runIvasCodec.py $verbosity_cmd -m $non_ism_modes -p $cfg $duration_arg $timeout_cmd \| tee smoke_test_output.txt
		echo "\n======================= 2. ism modes no FEC =======================\n\n"
		./scripts/runIvasCodec.py $verbosity_cmd -m $ism_modes -p $cfg $duration_arg $ism_md_cmd $timeout_cmd \| tee smoke_test_output.txt
		readarray -t ism_modes < <(./scripts/runIvasCodec.py -l \| grep ISM)
		readarray -t non_ism_modes < <(./scripts/runIvasCodec.py -l \| grep -v ISM)

		echo -e "\n======================= 1. non-ism modes no FEC =======================\n\n"
		./scripts/runIvasCodec.py $verbosity_cmd -m "${non_ism_modes[@]}" -p $cfg $duration_arg $timeout_cmd \| tee smoke_test_output.txt
		echo -e "\n======================= 2. ism modes no FEC =======================\n\n"
		./scripts/runIvasCodec.py $verbosity_cmd -m "${ism_modes[@]}" -p $cfg $duration_arg $ism_md_cmd $timeout_cmd \| tee smoke_test_output.txt

		# all modes with simulated network delay - this includes JBM TSM and lost frames
		echo "\n======================= 3. JBM =======================\n\n"
		echo -e "\n======================= 3. JBM =======================\n\n"
		./scripts/runIvasCodec.py $verbosity_cmd -p $cfg $duration_arg --decoder_only --jbm_file $dly_profile $timeout_cmd \| tee smoke_test_output_jbm.txt

		# run all modes with binaural output using external files
		formats_with_bin_out=$(./scripts/runIvasCodec.py -L \| grep -v "mono\\|tereo")
		bin_out_modes="BINAURAL BINAURAL_ROOM_IR BINAURAL_ROOM_REVERB"
		readarray -t formats_with_bin_out < <(./scripts/runIvasCodec.py -L \| grep -v "mono\\|tereo")
		bin_out_modes=(BINAURAL BINAURAL_ROOM_IR BINAURAL_ROOM_REVERB)

		echo "\n======================= 4. binaural out with HRTF files - WB =======================\n\n"
		wb_modes=$(./scripts/runIvasCodec.py -l -C $formats_with_bin_out \| grep _wb_)
		echo -e "\n======================= 4. binaural out with HRTF files - WB =======================\n\n"
		readarray -t wb_modes < <(./scripts/runIvasCodec.py -l -C "${formats_with_bin_out[@]}" \| grep _wb_)
		hrtf_wb="../scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_16kHz.bin"
		./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m $wb_modes $duration_arg -D="-hrtf ${hrtf_wb}" --decoder_only --oc $bin_out_modes $timeout_cmd \| tee -a smoke_test_output_hrtf.txt
		./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m "${wb_modes[@]}" $duration_arg -D="-hrtf ${hrtf_wb}" --decoder_only --oc "${bin_out_modes[@]}" $timeout_cmd \| tee -a smoke_test_output_hrtf.txt

		echo "\n======================= 5. binaural out with HRTF files - SWB =======================\n\n"
		swb_modes=$(./scripts/runIvasCodec.py -l -C $formats_with_bin_out \| grep _swb_)
		echo -e "\n======================= 5. binaural out with HRTF files - SWB =======================\n\n"
		readarray -t swb_modes < <(./scripts/runIvasCodec.py -l -C "${formats_with_bin_out[@]}" \| grep _swb_)
		hrtf_swb="../scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_32kHz.bin"
		./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m $swb_modes $duration_arg -D="-hrtf ${hrtf_swb}" --decoder_only --oc $bin_out_modes $timeout_cmd \| tee -a smoke_test_output_hrtf.txt
		./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m "${swb_modes[@]}" $duration_arg -D="-hrtf ${hrtf_swb}" --decoder_only --oc "${bin_out_modes[@]}" $timeout_cmd \| tee -a smoke_test_output_hrtf.txt

		echo "\n======================= 6. binaural out with HRTF files - FB =======================\n\n"
		fb_modes=$(./scripts/runIvasCodec.py -l -C $formats_with_bin_out \| grep _fb_)
		echo -e "\n======================= 6. binaural out with HRTF files - FB =======================\n\n"
		readarray -t fb_modes < <(./scripts/runIvasCodec.py -l -C "${formats_with_bin_out[@]}" \| grep _fb_)
		hrtf_fb="../scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_48kHz.bin"
		./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m $fb_modes $duration_arg -D="-hrtf ${hrtf_fb}" --decoder_only --oc $bin_out_modes $timeout_cmd \| tee -a smoke_test_output_hrtf.txt
		./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m "${fb_modes[@]}" $duration_arg -D="-hrtf ${hrtf_fb}" --decoder_only --oc "${bin_out_modes[@]}" $timeout_cmd \| tee -a smoke_test_output_hrtf.txt

scripts/pyaudio3dtools/audiofile.py

+3 −8

Original line number	Diff line number	Diff line
		@@ -676,9 +676,7 @@ def get_wav_file_info(filename: str) -> dict:

		"""

		fid = open(filename, "rb")

		try:
		with open(filename, "rb") as fid:
		riff = fid.read(4)

		if riff == b"RIFF":
		@@ -686,13 +684,13 @@ def get_wav_file_info(filename: str) -> dict:
		elif riff == b"RIFX":
		binary_format = ">"
		else:
		raise ValueError("No RIFF!")
		raise ValueError(f"Not a RIFF file: {filename}")

		wav_size = struct.unpack(f"{binary_format}I", fid.read(4))[0]

		wav_identifier = fid.read(4)
		if wav_identifier != b"WAVE":
		raise ValueError("No WAVE!")
		raise ValueError(f"Not a WAVE file: {filename}")

		fmt_chunk_id = fid.read(4)

		@@ -715,9 +713,6 @@ def get_wav_file_info(filename: str) -> dict:
		else:
		raise ValueError("No or corrupt fmt chunk!")

		finally:
		fid.close()

		return {
		"size": wav_size,
		"format_tag": wav_format,

scripts/pyivastest/IvasModeAnalyzer.py

+8 −9

Original line number	Diff line number	Diff line
		@@ -296,9 +296,8 @@ class IvasModeAnalyzer(IvasModeCollector):
		keywords = [keywords]

		if os.path.exists(filename):
		fp = open(filename)
		with open(filename) as fp:
		loglines = fp.readlines()
		fp.close()
		matching_lines = [
		line
		for line in loglines
		@@ -571,7 +570,7 @@ class IvasModeAnalyzer(IvasModeCollector):
		"""
		loglines = []
		if os.path.exists(filename):
		fp = open(filename)
		with open(filename) as fp:
		loglines = fp.readlines()
		vg_errors = {"errors": 0, "contexts": 0, "heap_bytes": 0, "heap_blocks": 0}
		for line in loglines:
		@@ -602,7 +601,7 @@ class IvasModeAnalyzer(IvasModeCollector):
		"""
		loglines = []
		if os.path.exists(filename):
		fp = open(filename)
		with open(filename) as fp:
		loglines = fp.readlines()
		build_errors = 0
		for line in loglines:
		@@ -1053,7 +1052,7 @@ class IvasModeAnalyzer(IvasModeCollector):
		def get_snr_report(file_name):
		SNR_report = {}
		if os.path.exists(file_name):
		fp = open(file_name, "r")
		with open(file_name) as fp:
		lines = fp.readlines()
		for line in lines:
		line = line.rstrip("\n")

scripts/pyivastest/IvasModeRunner.py

+376 −386

Original line number	Diff line number	Diff line
		@@ -242,7 +242,7 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):

		"""

		self.lock.acquire()
		with self.lock:
		if self.stats:
		if not self.run_encoder:
		if config["config"]["num_dec_remaining"] == config["config"]["num_dec"]:
		@@ -250,7 +250,6 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
		self.stats["num_decs_running"] += 1
		config["config"]["num_dec_remaining"] -= 1
		self.show_progress()
		self.lock.release()
		enc_dec_cmd = config["config"]["cmd"]
		item_base_name = os.path.splitext(os.path.basename(config["enc_file_name"]))[0]
		output_config = config["out_config"]
		@@ -261,7 +260,7 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
		dec_log_name = self.get_dec_log_file_name(
		self.dir_name, item_base_name, output_config
		)
		dec_log = open(dec_log_name, "w")
		with open(dec_log_name, "w") as dec_log:
		if config["enc_returncode"] == 0:
		dec_file_name = self.get_dec_file_name(
		self.dir_name, item_base_name, output_config, out_ext
		@@ -339,15 +338,15 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
		returncode = RET_CODE_TIMEOUT_EXP

		if returncode != 0:
		fail_string = "Decoding {} to {} failed!"
		fail_string = "Decoding {} to {} failed "
		if returncode == RET_CODE_TIMEOUT_EXP:
		fail_string = (
		fail_string[:-1]
		+ f" due to timeout after {self.timeout} seconds!"
		)
		fail_string += f"due to timeout after {self.timeout} seconds"
		else:
		fail_string += f"with exit code {returncode}"
		fail_string += "!"

		self.logger.error(fail_string.format(enc_file_name, dec_file_name))
		self.lock.acquire()
		with self.lock:
		if self.stats:
		self.stats["num_dec_errors"] += 1
		self.show_progress()
		@@ -370,7 +369,6 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
		self.failed_modes["dec"][output_config].append(mode)
		else:
		self.failed_modes["dec"][output_config] = [mode]
		self.lock.release()
		else:
		self.logger.info(
		"Decoding successful for {} to {}".format(
		@@ -388,8 +386,8 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
		config["enc_file_name"], output_config
		)
		)
		self.lock.acquire()

		with self.lock:
		config["config"]["num_dec_done"] += 1
		if self.stats:
		self.stats["num_decs_finished"] += 1
		@@ -401,8 +399,6 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
		self.stats["num_modes_finished"] += 1
		self.show_progress()

		self.lock.release()

		def clean_pcm_directory(self):
		"""
		Remove .LOCK files and corresponding intermediate files from the pcm directory
		@@ -439,8 +435,8 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
		error = 0
		enc_dec_cmd = deepcopy(config["cmd"])
		# get next item
		config["lock"].acquire()
		self.lock.acquire()
		with config["lock"]:
		with self.lock:
		if self.stats:
		if config["num_enc_remaining"] == config["num_enc"]:
		self.stats["num_modes_running"] += 1
		@@ -449,9 +445,7 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
		in_file_name = config["item_list"].pop()
		config["num_enc_remaining"] -= 1
		else:
		config["lock"].release()
		return
		self.lock.release()

		metadata_file_names = list()

		@@ -460,12 +454,11 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
		metadata_file_names = in_file_name[1:]
		in_file_name = in_file_name[0]
		self.logger.info("Encoding Mode {} input file {}".format(mode, in_file_name))
		config["lock"].release()
		self.lock.acquire()

		with self.lock:
		if self.stats:
		self.stats["num_encs_running"] += 1
		self.show_progress()
		self.lock.release()
		pcm_name_lock = None
		item_base_name = os.path.splitext(os.path.basename(in_file_name))[0]
		enc_file_name = os.path.join(
		@@ -524,14 +517,21 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
		pcm_name = os.path.join(self.dir_name, "pcm", pcm_base_name)
		pcm_name_lock = "".join([pcm_name, ".LOCK"])
		pcm_info_file = f"{pcm_name}.json"
		self.lock.acquire()

		create_input_file = False
		pcm_ready = False

		with self.lock:
		if not os.path.exists(pcm_name_lock):
		if not os.path.exists(pcm_name):
		# create the lock
		fp = open(pcm_name_lock, "w")
		fp.close()
		self.lock.release()
		# create lock file
		with open(pcm_name_lock, "w"):
		pass

		create_input_file = True

		if create_input_file:

		self.logger.info(
		"Creating input pcm for item {} at sample rate {}: {}".format(
		item_base_name, sample_rate_in, pcm_name
		@@ -544,8 +544,8 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
		pcm_log_name = os.path.join(
		self.dir_name, "logs", pcm_log_name_tmp
		)
		pcm_log = open(pcm_log_name, "w")

		with open(pcm_log_name, "w") as pcm_log:
		pcm_name_res_tmp = pcm_name + ".res.wav"
		pcm_name_cpy_tmp = pcm_name + ".cpy.wav"
		in_file_name_transformed = self.transform_path(in_file_name)
		@@ -623,10 +623,8 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):

		in_config = config["cmd"]["in_config"].upper()

		pcm_log.flush()
		pcm_log.close()
		# remove file lock
		self.lock.acquire()
		with self.lock:
		os.remove(pcm_name_lock)
		# os.remove(pcm_name_res_tmp)
		if do_limit_duration and cut_len_samples < in_len:
		@@ -634,26 +632,24 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
		self.logger.info(
		"PCM file {} successfully created!".format(pcm_name)
		)
		self.lock.release()
		else:
		self.lock.release()
		else:

		# PCM is now ready, no need to wait for anybody
		pcm_ready = True

		# wait for pcm to be ready
		self.lock.release()
		not_ready = 1
		while not_ready:
		while not pcm_ready:
		time.sleep(1)
		self.lock.acquire()

		with self.lock:
		if not os.path.exists(pcm_name_lock):
		not_ready = 0
		self.lock.release()
		pcm_ready = True

		else:
		pcm_name = in_file_name
		pcm_info_file = f"{pcm_name}.json"

		# get input format dictionary for the input file
		self.lock.acquire()
		with self.lock:
		if pcm_name not in self.pcm_info:
		if os.path.exists(pcm_info_file):
		with open(pcm_info_file, "r") as pcmifp:
		@@ -666,7 +662,6 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
		in_format = "HOA3"
		in_format_dict = spformat.get_format_dict(in_format)
		self.pcm_info.update({pcm_name: in_format_dict})
		self.lock.release()

		# build the encoder commandline
		enc_options = enc_dec_cmd["encmodeoption"]
		@@ -754,7 +749,7 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
		for x in enc_cmd
		]

		enc_log = open(enc_log_name, "w")
		with open(enc_log_name, "w") as enc_log:
		enc_log.write(" ".join(enc_cmd))
		try:
		enc_result = subprocess.run(
		@@ -788,7 +783,6 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
		x.format(out_file=bs_out_file) if "{out_file}" in x else x
		for x in proc_cmd
		]
		enc_log = open(enc_log_name, "a")
		enc_log.write(" ".join(proc_cmd))
		proc_result = subprocess.run(
		proc_cmd, capture_output=True, text=True, encoding="utf8"
		@@ -815,7 +809,7 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):

		except Exception as exc:
		# make sure we do not have a deadlock...
		self.lock.acquire()
		with self.lock:
		if pcm_name_lock:

		if os.path.exists(pcm_name_lock):
		@@ -836,11 +830,10 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
		"Traceback: {}".format(traceback.format_tb(exc.__traceback__)),
		logging.ERROR,
		)
		self.lock.release()
		enc_file_name_dec = enc_file_name
		error = 1

		self.lock.acquire()
		with self.lock:
		if self.stats:
		config["num_enc_done"] += 1
		if config["num_enc_done"] == config["num_enc"]:
		@@ -853,16 +846,15 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
		self.stats["num_encs_finished"] += 1
		self.stats["num_encs_running"] -= 1
		self.show_progress()
		self.lock.release()

		if error != 0:
		fail_string = "Encoding failed for {}"
		fail_string = "error " + str(error) + ": Encoding failed for {}"
		if error == RET_CODE_TIMEOUT_EXP:
		fail_string = (
		fail_string + f" due to timeout after {self.timeout} seconds"
		)

		self.lock.acquire()
		with self.lock:
		if self.stats:
		self.stats["num_enc_errors"] += 1
		self.show_progress()
		@@ -875,7 +867,6 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
		]
		)
		self.failed_modes["enc"].append(mode)
		self.lock.release()
		self.logger.error(fail_string.format(enc_file_name))
		else:
		self.logger.info("Encoding successful for {}".format(enc_file_name))
		@@ -1169,7 +1160,7 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
		return missing_mode_flat_dict

		def bs_processing_thread(self, bs_entry):
		self.lock.acquire()
		with self.lock:
		self.bs_processing_queue["stats"]["num_bs_running"] += 1
		line = "Bit stream processing: {}/{} ({} running)".format(
		self.bs_processing_queue["stats"]["num_bs_done"],
		@@ -1178,7 +1169,6 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
		)
		self.logger.progress(line)
		enc_file_name = bs_entry["item_list"].pop()
		self.lock.release()
		bs_in_file = enc_file_name
		proc_chain = deepcopy(bs_entry["proc_chain"])
		suffices = self.get_bs_processing_suffices(bs_entry)
		@@ -1195,7 +1185,8 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
		]
		),
		)
		enc_log = open(enc_log_name, "w")

		with open(enc_log_name, "w") as enc_log:
		self.logger.info("Processing bit stream {}".format(enc_file_name))
		try:
		for processing in proc_chain:
		@@ -1239,7 +1230,7 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
		self.logger.error(
		"Traceback: {}".format(traceback.format_tb(exc.__traceback__))
		)
		self.lock.acquire()
		with self.lock:
		self.bs_processing_queue["stats"]["num_bs_running"] -= 1
		self.bs_processing_queue["stats"]["num_bs_done"] += 1
		line = "Bit stream processing: {}/{} ({} running)".format(
		@@ -1248,7 +1239,6 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
		self.bs_processing_queue["stats"]["num_bs_running"],
		)
		self.logger.progress(line)
		self.lock.release()

		def run_bs_processing_queue(self):
		self.logger.console("Postprocessing of bit streams ...", logging.INFO)