diff --git a/ci/smoke_test.sh b/ci/smoke_test.sh index 071b9a0be620c8ebf8aef7c00f50914459088047..32e00dfba176e4d25be81e4ccc7e00b7585599e8 100755 --- a/ci/smoke_test.sh +++ b/ci/smoke_test.sh @@ -28,6 +28,8 @@ # accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and # the United Nations Convention on Contracts on the International Sales of Goods. +set -euxo pipefail + function usage { echo echo "Usage:" @@ -42,9 +44,9 @@ if [ ! -d "lib_com" ]; then exit 1 fi -if [ -z "$1" ] || [ "$1" == "test" ]; then +if [ -z "${1:-}" ] || [ "${1:-}" == "test" ]; then BUILD=1 -elif [ "$1" == "coverage" ]; then +elif [ "${1:-}" == "coverage" ]; then BUILD=0 else usage @@ -76,37 +78,38 @@ if [ $BUILD -eq 1 ];then fi # prepare combined format test signals -echo "\n======================= 0. preparing combined format test inputs =======================\n\n" +echo -e "\n======================= 0. preparing combined format test inputs =======================\n\n" ./scripts/prepare_combined_format_inputs.py # run all modes vanilla-fashion # treat ISM modes separately because passing the metadata files to MASA modes causes crashes -ism_modes=$(./scripts/runIvasCodec.py -l | grep ISM) -non_ism_modes=$(./scripts/runIvasCodec.py -l | grep -v ISM) -echo "\n======================= 1. non-ism modes no FEC =======================\n\n" -./scripts/runIvasCodec.py $verbosity_cmd -m $non_ism_modes -p $cfg $duration_arg $timeout_cmd | tee smoke_test_output.txt -echo "\n======================= 2. ism modes no FEC =======================\n\n" -./scripts/runIvasCodec.py $verbosity_cmd -m $ism_modes -p $cfg $duration_arg $ism_md_cmd $timeout_cmd | tee smoke_test_output.txt +readarray -t ism_modes < <(./scripts/runIvasCodec.py -l | grep ISM) +readarray -t non_ism_modes < <(./scripts/runIvasCodec.py -l | grep -v ISM) + +echo -e "\n======================= 1. non-ism modes no FEC =======================\n\n" +./scripts/runIvasCodec.py $verbosity_cmd -m "${non_ism_modes[@]}" -p $cfg $duration_arg $timeout_cmd | tee smoke_test_output.txt +echo -e "\n======================= 2. ism modes no FEC =======================\n\n" +./scripts/runIvasCodec.py $verbosity_cmd -m "${ism_modes[@]}" -p $cfg $duration_arg $ism_md_cmd $timeout_cmd | tee smoke_test_output.txt # all modes with simulated network delay - this includes JBM TSM and lost frames -echo "\n======================= 3. JBM =======================\n\n" +echo -e "\n======================= 3. JBM =======================\n\n" ./scripts/runIvasCodec.py $verbosity_cmd -p $cfg $duration_arg --decoder_only --jbm_file $dly_profile $timeout_cmd | tee smoke_test_output_jbm.txt # run all modes with binaural output using external files -formats_with_bin_out=$(./scripts/runIvasCodec.py -L | grep -v "mono\|tereo") -bin_out_modes="BINAURAL BINAURAL_ROOM_IR BINAURAL_ROOM_REVERB" +readarray -t formats_with_bin_out < <(./scripts/runIvasCodec.py -L | grep -v "mono\|tereo") +bin_out_modes=(BINAURAL BINAURAL_ROOM_IR BINAURAL_ROOM_REVERB) -echo "\n======================= 4. binaural out with HRTF files - WB =======================\n\n" -wb_modes=$(./scripts/runIvasCodec.py -l -C $formats_with_bin_out | grep _wb_) +echo -e "\n======================= 4. binaural out with HRTF files - WB =======================\n\n" +readarray -t wb_modes < <(./scripts/runIvasCodec.py -l -C "${formats_with_bin_out[@]}" | grep _wb_) hrtf_wb="../scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_16kHz.bin" -./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m $wb_modes $duration_arg -D="-hrtf ${hrtf_wb}" --decoder_only --oc $bin_out_modes $timeout_cmd | tee -a smoke_test_output_hrtf.txt +./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m "${wb_modes[@]}" $duration_arg -D="-hrtf ${hrtf_wb}" --decoder_only --oc "${bin_out_modes[@]}" $timeout_cmd | tee -a smoke_test_output_hrtf.txt -echo "\n======================= 5. binaural out with HRTF files - SWB =======================\n\n" -swb_modes=$(./scripts/runIvasCodec.py -l -C $formats_with_bin_out | grep _swb_) +echo -e "\n======================= 5. binaural out with HRTF files - SWB =======================\n\n" +readarray -t swb_modes < <(./scripts/runIvasCodec.py -l -C "${formats_with_bin_out[@]}" | grep _swb_) hrtf_swb="../scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_32kHz.bin" -./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m $swb_modes $duration_arg -D="-hrtf ${hrtf_swb}" --decoder_only --oc $bin_out_modes $timeout_cmd | tee -a smoke_test_output_hrtf.txt +./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m "${swb_modes[@]}" $duration_arg -D="-hrtf ${hrtf_swb}" --decoder_only --oc "${bin_out_modes[@]}" $timeout_cmd | tee -a smoke_test_output_hrtf.txt -echo "\n======================= 6. binaural out with HRTF files - FB =======================\n\n" -fb_modes=$(./scripts/runIvasCodec.py -l -C $formats_with_bin_out | grep _fb_) +echo -e "\n======================= 6. binaural out with HRTF files - FB =======================\n\n" +readarray -t fb_modes < <(./scripts/runIvasCodec.py -l -C "${formats_with_bin_out[@]}" | grep _fb_) hrtf_fb="../scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_48kHz.bin" -./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m $fb_modes $duration_arg -D="-hrtf ${hrtf_fb}" --decoder_only --oc $bin_out_modes $timeout_cmd | tee -a smoke_test_output_hrtf.txt +./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m "${fb_modes[@]}" $duration_arg -D="-hrtf ${hrtf_fb}" --decoder_only --oc "${bin_out_modes[@]}" $timeout_cmd | tee -a smoke_test_output_hrtf.txt diff --git a/scripts/pyaudio3dtools/audiofile.py b/scripts/pyaudio3dtools/audiofile.py index bbee88ca9c52f570489858bf5c779c5ed4a06984..a0af594f40af0376970d18f6a034a7f0c7dba02e 100644 --- a/scripts/pyaudio3dtools/audiofile.py +++ b/scripts/pyaudio3dtools/audiofile.py @@ -676,9 +676,7 @@ def get_wav_file_info(filename: str) -> dict: """ - fid = open(filename, "rb") - - try: + with open(filename, "rb") as fid: riff = fid.read(4) if riff == b"RIFF": @@ -686,13 +684,13 @@ def get_wav_file_info(filename: str) -> dict: elif riff == b"RIFX": binary_format = ">" else: - raise ValueError("No RIFF!") + raise ValueError(f"Not a RIFF file: {filename}") wav_size = struct.unpack(f"{binary_format}I", fid.read(4))[0] wav_identifier = fid.read(4) if wav_identifier != b"WAVE": - raise ValueError("No WAVE!") + raise ValueError(f"Not a WAVE file: {filename}") fmt_chunk_id = fid.read(4) @@ -715,9 +713,6 @@ def get_wav_file_info(filename: str) -> dict: else: raise ValueError("No or corrupt fmt chunk!") - finally: - fid.close() - return { "size": wav_size, "format_tag": wav_format, diff --git a/scripts/pyivastest/IvasModeAnalyzer.py b/scripts/pyivastest/IvasModeAnalyzer.py index 91bf0b44ae92cac34e8fdcc13313885628df155f..e73d7fbcb5403cc0b866db62e24f37f3dbc5c3ab 100644 --- a/scripts/pyivastest/IvasModeAnalyzer.py +++ b/scripts/pyivastest/IvasModeAnalyzer.py @@ -296,9 +296,8 @@ class IvasModeAnalyzer(IvasModeCollector): keywords = [keywords] if os.path.exists(filename): - fp = open(filename) - loglines = fp.readlines() - fp.close() + with open(filename) as fp: + loglines = fp.readlines() matching_lines = [ line for line in loglines @@ -571,8 +570,8 @@ class IvasModeAnalyzer(IvasModeCollector): """ loglines = [] if os.path.exists(filename): - fp = open(filename) - loglines = fp.readlines() + with open(filename) as fp: + loglines = fp.readlines() vg_errors = {"errors": 0, "contexts": 0, "heap_bytes": 0, "heap_blocks": 0} for line in loglines: # print(line) @@ -602,8 +601,8 @@ class IvasModeAnalyzer(IvasModeCollector): """ loglines = [] if os.path.exists(filename): - fp = open(filename) - loglines = fp.readlines() + with open(filename) as fp: + loglines = fp.readlines() build_errors = 0 for line in loglines: # print(line) @@ -1053,8 +1052,8 @@ class IvasModeAnalyzer(IvasModeCollector): def get_snr_report(file_name): SNR_report = {} if os.path.exists(file_name): - fp = open(file_name, "r") - lines = fp.readlines() + with open(file_name) as fp: + lines = fp.readlines() for line in lines: line = line.rstrip("\n") if re.findall("SNR report", line): diff --git a/scripts/pyivastest/IvasModeRunner.py b/scripts/pyivastest/IvasModeRunner.py index 41dd4d20ab0f296d72b204c6482a50c955e689ed..546cc2e82cbd16b727264c611eaaadd7f799bfdf 100644 --- a/scripts/pyivastest/IvasModeRunner.py +++ b/scripts/pyivastest/IvasModeRunner.py @@ -242,15 +242,14 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): """ - self.lock.acquire() - if self.stats: - if not self.run_encoder: - if config["config"]["num_dec_remaining"] == config["config"]["num_dec"]: - self.stats["num_modes_running"] += 1 - self.stats["num_decs_running"] += 1 - config["config"]["num_dec_remaining"] -= 1 - self.show_progress() - self.lock.release() + with self.lock: + if self.stats: + if not self.run_encoder: + if config["config"]["num_dec_remaining"] == config["config"]["num_dec"]: + self.stats["num_modes_running"] += 1 + self.stats["num_decs_running"] += 1 + config["config"]["num_dec_remaining"] -= 1 + self.show_progress() enc_dec_cmd = config["config"]["cmd"] item_base_name = os.path.splitext(os.path.basename(config["enc_file_name"]))[0] output_config = config["out_config"] @@ -261,147 +260,144 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): dec_log_name = self.get_dec_log_file_name( self.dir_name, item_base_name, output_config ) - dec_log = open(dec_log_name, "w") - if config["enc_returncode"] == 0: - dec_file_name = self.get_dec_file_name( - self.dir_name, item_base_name, output_config, out_ext - ) - if self.sample_rate_dec_out: - out_sr = self.sample_rate_dec_out - else: - if ( - isinstance(config["dec_options"], dict) - and "fs_out" in config["dec_options"] - ): - out_sr = config["dec_options"]["fs_out"] - else: - out_sr = BW_TO_SR[config["config"]["cmd"]["bw"]] - dec_options = [] - if isinstance(config["dec_options"], dict): - if "dec_options" in config["dec_options"]: - dec_options.extend(config["dec_options"]["dec_options"]) - else: - dec_options.extend(config["dec_options"]) - dec_options.extend(self.decoder_cmdline_options) - - if self.multiple_res_dir is True: - dec_options.extend(["-info", os.path.basename(dec_file_name)]) - - # handle SOFA file passing for group B binaural renderer - try: - br_kbps = int(config["config"]["cmd"]["bitrate"]) // 1000 - except ValueError: - # this handles the case when a bitrate switching file is given - # for now do nothing here - br_kbps = -1 - except Exception as e: - self.logger.error("Exception in ivas_dec_thread(): " + str(e)) - return - - if not config["mono"]: - dec_options.extend([output_config]) - dec_options = [ - x.format(dec_file_name=dec_file_name) if "{dec_file_name}" in x else x - for x in dec_options - if x != [] - ] - enc_file_name = config["enc_file_name"] - self.logger.info("Decoding {} to {}".format(enc_file_name, dec_file_name)) - if self.test_tool != "": - dec_cmd = ( - self.test_tool - + [self.decoder] - + dec_options - + [str(out_sr)] - + [enc_file_name, dec_file_name] - ) - else: - dec_cmd = ( - [self.decoder] - + dec_options - + [str(out_sr)] - + [enc_file_name, dec_file_name] + with open(dec_log_name, "w") as dec_log: + if config["enc_returncode"] == 0: + dec_file_name = self.get_dec_file_name( + self.dir_name, item_base_name, output_config, out_ext ) + if self.sample_rate_dec_out: + out_sr = self.sample_rate_dec_out + else: + if ( + isinstance(config["dec_options"], dict) + and "fs_out" in config["dec_options"] + ): + out_sr = config["dec_options"]["fs_out"] + else: + out_sr = BW_TO_SR[config["config"]["cmd"]["bw"]] + dec_options = [] + if isinstance(config["dec_options"], dict): + if "dec_options" in config["dec_options"]: + dec_options.extend(config["dec_options"]["dec_options"]) + else: + dec_options.extend(config["dec_options"]) + dec_options.extend(self.decoder_cmdline_options) + + if self.multiple_res_dir is True: + dec_options.extend(["-info", os.path.basename(dec_file_name)]) + + # handle SOFA file passing for group B binaural renderer + try: + br_kbps = int(config["config"]["cmd"]["bitrate"]) // 1000 + except ValueError: + # this handles the case when a bitrate switching file is given + # for now do nothing here + br_kbps = -1 + except Exception as e: + self.logger.error("Exception in ivas_dec_thread(): " + str(e)) + return + + if not config["mono"]: + dec_options.extend([output_config]) + dec_options = [ + x.format(dec_file_name=dec_file_name) if "{dec_file_name}" in x else x + for x in dec_options + if x != [] + ] + enc_file_name = config["enc_file_name"] + self.logger.info("Decoding {} to {}".format(enc_file_name, dec_file_name)) + if self.test_tool != "": + dec_cmd = ( + self.test_tool + + [self.decoder] + + dec_options + + [str(out_sr)] + + [enc_file_name, dec_file_name] + ) + else: + dec_cmd = ( + [self.decoder] + + dec_options + + [str(out_sr)] + + [enc_file_name, dec_file_name] + ) - dec_log.write(" ".join(dec_cmd)) - try: - cur_dec_result = subprocess.run( - dec_cmd, - capture_output=True, - text=True, - env=self.run_env, - timeout=self.timeout, - ) - returncode = cur_dec_result.returncode - dec_log.write(cur_dec_result.stderr) - dec_log.write(cur_dec_result.stdout) - except subprocess.TimeoutExpired: - returncode = RET_CODE_TIMEOUT_EXP - - if returncode != 0: - fail_string = "Decoding {} to {} failed!" - if returncode == RET_CODE_TIMEOUT_EXP: - fail_string = ( - fail_string[:-1] - + f" due to timeout after {self.timeout} seconds!" + dec_log.write(" ".join(dec_cmd)) + try: + cur_dec_result = subprocess.run( + dec_cmd, + capture_output=True, + text=True, + env=self.run_env, + timeout=self.timeout, ) + returncode = cur_dec_result.returncode + dec_log.write(cur_dec_result.stderr) + dec_log.write(cur_dec_result.stdout) + except subprocess.TimeoutExpired: + returncode = RET_CODE_TIMEOUT_EXP + + if returncode != 0: + fail_string = "Decoding {} to {} failed " + if returncode == RET_CODE_TIMEOUT_EXP: + fail_string += f"due to timeout after {self.timeout} seconds" + else: + fail_string += f"with exit code {returncode}" + fail_string += "!" - self.logger.error(fail_string.format(enc_file_name, dec_file_name)) - self.lock.acquire() - if self.stats: - self.stats["num_dec_errors"] += 1 - self.show_progress() - self.results.append( - [ - fail_string.format(output_config, enc_file_name), - config["ivas_format"], - enc_dec_cmd["table_name"], - dec_log_name, - ] - ) + self.logger.error(fail_string.format(enc_file_name, dec_file_name)) + with self.lock: + if self.stats: + self.stats["num_dec_errors"] += 1 + self.show_progress() + self.results.append( + [ + fail_string.format(output_config, enc_file_name), + config["ivas_format"], + enc_dec_cmd["table_name"], + dec_log_name, + ] + ) - # get mode from enc_file_name - file_base_no_ext = os.path.basename(enc_file_name)[ - :-4 - ] # NOTE: this assumes that all encoded files still end in ".192" - in_config = config["config"]["cmd"]["in_config"] - mode = file_base_no_ext[file_base_no_ext.rfind(in_config) :] - if output_config in self.failed_modes["dec"]: - self.failed_modes["dec"][output_config].append(mode) + # get mode from enc_file_name + file_base_no_ext = os.path.basename(enc_file_name)[ + :-4 + ] # NOTE: this assumes that all encoded files still end in ".192" + in_config = config["config"]["cmd"]["in_config"] + mode = file_base_no_ext[file_base_no_ext.rfind(in_config) :] + if output_config in self.failed_modes["dec"]: + self.failed_modes["dec"][output_config].append(mode) + else: + self.failed_modes["dec"][output_config] = [mode] else: - self.failed_modes["dec"][output_config] = [mode] - self.lock.release() + self.logger.info( + "Decoding successful for {} to {}".format( + enc_file_name, dec_file_name + ) + ) else: - self.logger.info( - "Decoding successful for {} to {}".format( - enc_file_name, dec_file_name + self.logger.error( + "No decoding of {} to {}, encoding failed already".format( + config["enc_file_name"], output_config ) ) - else: - self.logger.error( - "No decoding of {} to {}, encoding failed already".format( - config["enc_file_name"], output_config - ) - ) - dec_log.write( - "No decoding of {} to {}, encoding failed already".format( - config["enc_file_name"], output_config + dec_log.write( + "No decoding of {} to {}, encoding failed already".format( + config["enc_file_name"], output_config + ) ) - ) - self.lock.acquire() - - config["config"]["num_dec_done"] += 1 - if self.stats: - self.stats["num_decs_finished"] += 1 - self.stats["num_decs_running"] -= 1 - if config["enc_returncode"] != 0: - self.stats["num_dec_errors"] += 1 - if config["config"]["num_dec_done"] == config["config"]["num_dec"]: - self.stats["num_modes_running"] -= 1 - self.stats["num_modes_finished"] += 1 - self.show_progress() - self.lock.release() + with self.lock: + config["config"]["num_dec_done"] += 1 + if self.stats: + self.stats["num_decs_finished"] += 1 + self.stats["num_decs_running"] -= 1 + if config["enc_returncode"] != 0: + self.stats["num_dec_errors"] += 1 + if config["config"]["num_dec_done"] == config["config"]["num_dec"]: + self.stats["num_modes_running"] -= 1 + self.stats["num_modes_finished"] += 1 + self.show_progress() def clean_pcm_directory(self): """ @@ -439,33 +435,30 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): error = 0 enc_dec_cmd = deepcopy(config["cmd"]) # get next item - config["lock"].acquire() - self.lock.acquire() - if self.stats: - if config["num_enc_remaining"] == config["num_enc"]: - self.stats["num_modes_running"] += 1 + with config["lock"]: + with self.lock: + if self.stats: + if config["num_enc_remaining"] == config["num_enc"]: + self.stats["num_modes_running"] += 1 + self.show_progress() + if config["num_enc_remaining"]: + in_file_name = config["item_list"].pop() + config["num_enc_remaining"] -= 1 + else: + return + + metadata_file_names = list() + + # TODO: is this still needed - what case does this cover? + if isinstance(in_file_name, list): + metadata_file_names = in_file_name[1:] + in_file_name = in_file_name[0] + self.logger.info("Encoding Mode {} input file {}".format(mode, in_file_name)) + + with self.lock: + if self.stats: + self.stats["num_encs_running"] += 1 self.show_progress() - if config["num_enc_remaining"]: - in_file_name = config["item_list"].pop() - config["num_enc_remaining"] -= 1 - else: - config["lock"].release() - return - self.lock.release() - - metadata_file_names = list() - - # TODO: is this still needed - what case does this cover? - if isinstance(in_file_name, list): - metadata_file_names = in_file_name[1:] - in_file_name = in_file_name[0] - self.logger.info("Encoding Mode {} input file {}".format(mode, in_file_name)) - config["lock"].release() - self.lock.acquire() - if self.stats: - self.stats["num_encs_running"] += 1 - self.show_progress() - self.lock.release() pcm_name_lock = None item_base_name = os.path.splitext(os.path.basename(in_file_name))[0] enc_file_name = os.path.join( @@ -524,28 +517,35 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): pcm_name = os.path.join(self.dir_name, "pcm", pcm_base_name) pcm_name_lock = "".join([pcm_name, ".LOCK"]) pcm_info_file = f"{pcm_name}.json" - self.lock.acquire() - - if not os.path.exists(pcm_name_lock): - if not os.path.exists(pcm_name): - # create the lock - fp = open(pcm_name_lock, "w") - fp.close() - self.lock.release() - self.logger.info( - "Creating input pcm for item {} at sample rate {}: {}".format( - item_base_name, sample_rate_in, pcm_name - ) - ) - pcm_log_name_tmp = "".join( - [pcm_base_name, constants.LOG_FILE_EXT] - ) - pcm_log_name = os.path.join( - self.dir_name, "logs", pcm_log_name_tmp + create_input_file = False + pcm_ready = False + + with self.lock: + if not os.path.exists(pcm_name_lock): + if not os.path.exists(pcm_name): + # create lock file + with open(pcm_name_lock, "w"): + pass + + create_input_file = True + + if create_input_file: + + self.logger.info( + "Creating input pcm for item {} at sample rate {}: {}".format( + item_base_name, sample_rate_in, pcm_name ) - pcm_log = open(pcm_log_name, "w") + ) + pcm_log_name_tmp = "".join( + [pcm_base_name, constants.LOG_FILE_EXT] + ) + pcm_log_name = os.path.join( + self.dir_name, "logs", pcm_log_name_tmp + ) + + with open(pcm_log_name, "w") as pcm_log: pcm_name_res_tmp = pcm_name + ".res.wav" pcm_name_cpy_tmp = pcm_name + ".cpy.wav" in_file_name_transformed = self.transform_path(in_file_name) @@ -594,8 +594,8 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): cut_len_samples = end_time_samples - start_time_samples if ( - cut_len_samples + start_time_samples < in_len - or start_time_samples > 0 + cut_len_samples + start_time_samples < in_len + or start_time_samples > 0 ): end_time_samples = min(end_time_samples, in_len) sig = ar.cut( @@ -623,50 +623,45 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): in_config = config["cmd"]["in_config"].upper() - pcm_log.flush() - pcm_log.close() # remove file lock - self.lock.acquire() - os.remove(pcm_name_lock) - # os.remove(pcm_name_res_tmp) - if do_limit_duration and cut_len_samples < in_len: - os.remove(pcm_name_cpy_tmp) - self.logger.info( - "PCM file {} successfully created!".format(pcm_name) - ) - self.lock.release() - else: - self.lock.release() - else: - # wait for pcm to be ready - self.lock.release() - not_ready = 1 - while not_ready: - time.sleep(1) - self.lock.acquire() + with self.lock: + os.remove(pcm_name_lock) + # os.remove(pcm_name_res_tmp) + if do_limit_duration and cut_len_samples < in_len: + os.remove(pcm_name_cpy_tmp) + self.logger.info( + "PCM file {} successfully created!".format(pcm_name) + ) + + # PCM is now ready, no need to wait for anybody + pcm_ready = True + + # wait for pcm to be ready + while not pcm_ready: + time.sleep(1) + + with self.lock: if not os.path.exists(pcm_name_lock): - not_ready = 0 - self.lock.release() + pcm_ready = True else: pcm_name = in_file_name pcm_info_file = f"{pcm_name}.json" # get input format dictionary for the input file - self.lock.acquire() - if pcm_name not in self.pcm_info: - if os.path.exists(pcm_info_file): - with open(pcm_info_file, "r") as pcmifp: - in_format_dict = json.load(pcmifp) + with self.lock: + if pcm_name not in self.pcm_info: + if os.path.exists(pcm_info_file): + with open(pcm_info_file, "r") as pcmifp: + in_format_dict = json.load(pcmifp) - else: - # add in_format_dict according to the in_format of the mode... - if in_format == "SBA": - # use HOA3 as default for SBA if nothing more is known... - in_format = "HOA3" - in_format_dict = spformat.get_format_dict(in_format) - self.pcm_info.update({pcm_name: in_format_dict}) - self.lock.release() + else: + # add in_format_dict according to the in_format of the mode... + if in_format == "SBA": + # use HOA3 as default for SBA if nothing more is known... + in_format = "HOA3" + in_format_dict = spformat.get_format_dict(in_format) + self.pcm_info.update({pcm_name: in_format_dict}) # build the encoder commandline enc_options = enc_dec_cmd["encmodeoption"] @@ -754,128 +749,124 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): for x in enc_cmd ] - enc_log = open(enc_log_name, "w") - enc_log.write(" ".join(enc_cmd)) - try: - enc_result = subprocess.run( - enc_cmd, - capture_output=True, - text=True, - env=self.run_env, - timeout=self.timeout, - ) - error = enc_result.returncode - enc_log.write(enc_result.stderr) - enc_log.write(enc_result.stdout) - except subprocess.TimeoutExpired: - error = RET_CODE_TIMEOUT_EXP - - if error == 0 and "bitstream_processing" in enc_dec_cmd: - bs_in_file = enc_file_name - proc_chain = deepcopy(enc_dec_cmd["bitstream_processing"]["proc_chain"]) - for processing in proc_chain: - suffix = processing.pop() - bs_out_file = ".".join( - ["_".join([os.path.splitext(bs_in_file)[0], suffix]), "192"] - ) - # process - proc_cmd = deepcopy(processing) - proc_cmd = [ - x.format(in_file=bs_in_file) if "{in_file}" in x else x - for x in proc_cmd - ] - proc_cmd = [ - x.format(out_file=bs_out_file) if "{out_file}" in x else x - for x in proc_cmd - ] - enc_log = open(enc_log_name, "a") - enc_log.write(" ".join(proc_cmd)) - proc_result = subprocess.run( - proc_cmd, capture_output=True, text=True, encoding="utf8" + with open(enc_log_name, "w") as enc_log: + enc_log.write(" ".join(enc_cmd)) + try: + enc_result = subprocess.run( + enc_cmd, + capture_output=True, + text=True, + env=self.run_env, + timeout=self.timeout, ) - enc_log.write(proc_result.stderr) - enc_log.write(proc_result.stdout) - error = proc_result.returncode - if error: - self.logger.error( - "Processing step {} for {} failed!".format( - suffix, enc_file_name - ) + error = enc_result.returncode + enc_log.write(enc_result.stderr) + enc_log.write(enc_result.stdout) + except subprocess.TimeoutExpired: + error = RET_CODE_TIMEOUT_EXP + + if error == 0 and "bitstream_processing" in enc_dec_cmd: + bs_in_file = enc_file_name + proc_chain = deepcopy(enc_dec_cmd["bitstream_processing"]["proc_chain"]) + for processing in proc_chain: + suffix = processing.pop() + bs_out_file = ".".join( + ["_".join([os.path.splitext(bs_in_file)[0], suffix]), "192"] ) - self.logger.error(proc_result.stderr) - raise RuntimeError( - "Processing step {} for {} failed!\n{}".format( - suffix, enc_file_name, proc_result.stderr - ) + # process + proc_cmd = deepcopy(processing) + proc_cmd = [ + x.format(in_file=bs_in_file) if "{in_file}" in x else x + for x in proc_cmd + ] + proc_cmd = [ + x.format(out_file=bs_out_file) if "{out_file}" in x else x + for x in proc_cmd + ] + enc_log.write(" ".join(proc_cmd)) + proc_result = subprocess.run( + proc_cmd, capture_output=True, text=True, encoding="utf8" ) - bs_in_file = bs_out_file - enc_file_name_dec = bs_out_file - else: - enc_file_name_dec = enc_file_name + enc_log.write(proc_result.stderr) + enc_log.write(proc_result.stdout) + error = proc_result.returncode + if error: + self.logger.error( + "Processing step {} for {} failed!".format( + suffix, enc_file_name + ) + ) + self.logger.error(proc_result.stderr) + raise RuntimeError( + "Processing step {} for {} failed!\n{}".format( + suffix, enc_file_name, proc_result.stderr + ) + ) + bs_in_file = bs_out_file + enc_file_name_dec = bs_out_file + else: + enc_file_name_dec = enc_file_name except Exception as exc: # make sure we do not have a deadlock... - self.lock.acquire() - if pcm_name_lock: - - if os.path.exists(pcm_name_lock): - # something went wrong with PCM creation, so make sure all waiting threads can continue... - self.logger.error( - "Something went wrong with creating the PCM file {}".format( - pcm_name + with self.lock: + if pcm_name_lock: + + if os.path.exists(pcm_name_lock): + # something went wrong with PCM creation, so make sure all waiting threads can continue... + self.logger.error( + "Something went wrong with creating the PCM file {}".format( + pcm_name + ) ) - ) - os.remove(pcm_name_lock) - os.remove(pcm_name) + os.remove(pcm_name_lock) + os.remove(pcm_name) - self.logger.console( - "Exception when encoding item {}: {}".format(enc_file_name, str(exc)), - logging.ERROR, - ) - self.logger.console( - "Traceback: {}".format(traceback.format_tb(exc.__traceback__)), - logging.ERROR, - ) - self.lock.release() + self.logger.console( + "Exception when encoding item {}: {}".format(enc_file_name, str(exc)), + logging.ERROR, + ) + self.logger.console( + "Traceback: {}".format(traceback.format_tb(exc.__traceback__)), + logging.ERROR, + ) enc_file_name_dec = enc_file_name error = 1 - self.lock.acquire() - if self.stats: - config["num_enc_done"] += 1 - if config["num_enc_done"] == config["num_enc"]: - self.enc_queue["num_modes_enc_done"] += 1 - if not self.run_decoder: - self.stats["num_modes_running"] -= 1 - self.stats["num_modes_finished"] -= 1 - if self.enc_queue["num_modes_enc"] == self.enc_queue["num_modes_enc_done"]: - self.dec_queue["all_encoded"] = True - self.stats["num_encs_finished"] += 1 - self.stats["num_encs_running"] -= 1 - self.show_progress() - self.lock.release() + with self.lock: + if self.stats: + config["num_enc_done"] += 1 + if config["num_enc_done"] == config["num_enc"]: + self.enc_queue["num_modes_enc_done"] += 1 + if not self.run_decoder: + self.stats["num_modes_running"] -= 1 + self.stats["num_modes_finished"] -= 1 + if self.enc_queue["num_modes_enc"] == self.enc_queue["num_modes_enc_done"]: + self.dec_queue["all_encoded"] = True + self.stats["num_encs_finished"] += 1 + self.stats["num_encs_running"] -= 1 + self.show_progress() if error != 0: - fail_string = "Encoding failed for {}" + fail_string = "error " + str(error) + ": Encoding failed for {}" if error == RET_CODE_TIMEOUT_EXP: fail_string = ( fail_string + f" due to timeout after {self.timeout} seconds" ) - self.lock.acquire() - if self.stats: - self.stats["num_enc_errors"] += 1 - self.show_progress() - self.results.append( - [ - fail_string.format(enc_file_name), - mode, - enc_dec_cmd["table_name"], - enc_log_name, - ] - ) - self.failed_modes["enc"].append(mode) - self.lock.release() + with self.lock: + if self.stats: + self.stats["num_enc_errors"] += 1 + self.show_progress() + self.results.append( + [ + fail_string.format(enc_file_name), + mode, + enc_dec_cmd["table_name"], + enc_log_name, + ] + ) + self.failed_modes["enc"].append(mode) self.logger.error(fail_string.format(enc_file_name)) else: self.logger.info("Encoding successful for {}".format(enc_file_name)) @@ -1169,16 +1160,15 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): return missing_mode_flat_dict def bs_processing_thread(self, bs_entry): - self.lock.acquire() - self.bs_processing_queue["stats"]["num_bs_running"] += 1 - line = "Bit stream processing: {}/{} ({} running)".format( - self.bs_processing_queue["stats"]["num_bs_done"], - self.bs_processing_queue["stats"]["num_bs_total"], - self.bs_processing_queue["stats"]["num_bs_running"], - ) - self.logger.progress(line) - enc_file_name = bs_entry["item_list"].pop() - self.lock.release() + with self.lock: + self.bs_processing_queue["stats"]["num_bs_running"] += 1 + line = "Bit stream processing: {}/{} ({} running)".format( + self.bs_processing_queue["stats"]["num_bs_done"], + self.bs_processing_queue["stats"]["num_bs_total"], + self.bs_processing_queue["stats"]["num_bs_running"], + ) + self.logger.progress(line) + enc_file_name = bs_entry["item_list"].pop() bs_in_file = enc_file_name proc_chain = deepcopy(bs_entry["proc_chain"]) suffices = self.get_bs_processing_suffices(bs_entry) @@ -1195,60 +1185,60 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector): ] ), ) - enc_log = open(enc_log_name, "w") - self.logger.info("Processing bit stream {}".format(enc_file_name)) - try: - for processing in proc_chain: - suffix = processing.pop() - bs_out_file = ".".join( - ["_".join([os.path.splitext(bs_in_file)[0], suffix]), "192"] - ) - # process - proc_cmd = deepcopy(processing) - proc_cmd = [ - x.format(in_file=bs_in_file) if "{in_file}" in x else x - for x in proc_cmd - ] - proc_cmd = [ - x.format(out_file=bs_out_file) if "{out_file}" in x else x - for x in proc_cmd - ] - enc_log.write(" ".join(proc_cmd)) - proc_result = subprocess.run(proc_cmd, capture_output=True, text=True) - enc_log.write(proc_result.stderr) - enc_log.write(proc_result.stdout) - error = proc_result.returncode - if error: - self.logger.error( - "Processing step {} for {} failed!".format( - suffix, enc_file_name - ) + with open(enc_log_name, "w") as enc_log: + self.logger.info("Processing bit stream {}".format(enc_file_name)) + try: + for processing in proc_chain: + suffix = processing.pop() + bs_out_file = ".".join( + ["_".join([os.path.splitext(bs_in_file)[0], suffix]), "192"] ) - raise RuntimeError( - "Processing step {} for {} failed!".format( - suffix, enc_file_name + # process + proc_cmd = deepcopy(processing) + proc_cmd = [ + x.format(in_file=bs_in_file) if "{in_file}" in x else x + for x in proc_cmd + ] + proc_cmd = [ + x.format(out_file=bs_out_file) if "{out_file}" in x else x + for x in proc_cmd + ] + + enc_log.write(" ".join(proc_cmd)) + proc_result = subprocess.run(proc_cmd, capture_output=True, text=True) + enc_log.write(proc_result.stderr) + enc_log.write(proc_result.stdout) + error = proc_result.returncode + if error: + self.logger.error( + "Processing step {} for {} failed!".format( + suffix, enc_file_name + ) ) - ) - bs_in_file = bs_out_file - except Exception as exc: - self.logger.error( - "Exception processing bitstream {}: {}".format(enc_file_name, str(exc)) - ) - self.logger.error("failed command: {}".format(" ".join(proc_cmd))) - self.logger.error( - "Traceback: {}".format(traceback.format_tb(exc.__traceback__)) - ) - self.lock.acquire() - self.bs_processing_queue["stats"]["num_bs_running"] -= 1 - self.bs_processing_queue["stats"]["num_bs_done"] += 1 - line = "Bit stream processing: {}/{} ({} running)".format( - self.bs_processing_queue["stats"]["num_bs_done"], - self.bs_processing_queue["stats"]["num_bs_total"], - self.bs_processing_queue["stats"]["num_bs_running"], - ) - self.logger.progress(line) - self.lock.release() + raise RuntimeError( + "Processing step {} for {} failed!".format( + suffix, enc_file_name + ) + ) + bs_in_file = bs_out_file + except Exception as exc: + self.logger.error( + "Exception processing bitstream {}: {}".format(enc_file_name, str(exc)) + ) + self.logger.error("failed command: {}".format(" ".join(proc_cmd))) + self.logger.error( + "Traceback: {}".format(traceback.format_tb(exc.__traceback__)) + ) + with self.lock: + self.bs_processing_queue["stats"]["num_bs_running"] -= 1 + self.bs_processing_queue["stats"]["num_bs_done"] += 1 + line = "Bit stream processing: {}/{} ({} running)".format( + self.bs_processing_queue["stats"]["num_bs_done"], + self.bs_processing_queue["stats"]["num_bs_total"], + self.bs_processing_queue["stats"]["num_bs_running"], + ) + self.logger.progress(line) def run_bs_processing_queue(self): self.logger.console("Postprocessing of bit streams ...", logging.INFO)