diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6d0d8bf458c511f837ad9e1deec6b358cde0bfa3..4f53b30fcd949330bf44268ee8e2e0bf3415c81d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -466,6 +466,7 @@ branch-is-up-to-date-with-main-post:
before_script:
- python3 ci/remove_unsupported_testcases.py $PARAM_FILE
- python3 tests/create_short_testvectors.py
+ - python3 scripts/prepare_combined_format_inputs.py
- *update-ltv-repo
- *copy-ltv-files-to-testv-dir
diff --git a/ci/basop-pages/create_summary_page.py b/ci/basop-pages/create_summary_page.py
index f8ddb00ae8fb23f38bc55315e1dae05fd7d6e024..1c2b46480213f24913e7288aa2659a828f1ec1e6 100644
--- a/ci/basop-pages/create_summary_page.py
+++ b/ci/basop-pages/create_summary_page.py
@@ -1,8 +1,16 @@
import argparse
+from typing import List
from create_report_pages import SUBPAGE_TMPL_CSS, FORMATS
-MEASURES = ["MLD","DIFF","SSNR","ODG","DELTA_ODG"]
+
+title = {
+ "MLD": "Maximum MLD across channels",
+ "DIFF": "Maximim absolute difference across channels",
+ "SSNR": "Minimum SSNR across channels",
+ "ODG": "Minimum PEAQ ODG across channels",
+ "DELTA_ODG": "PEAQ ODG using binauralized input and output",
+}
SUMMARY_PAGE_TMPL_HTML = """
@@ -12,12 +20,14 @@ SUMMARY_PAGE_TMPL_HTML = """
"""
+
def create_summary_page(
html_out,
id_current: int,
job_name: str,
+ measures: List[str],
):
- images = histogram_summary(job_name)
+ images = histogram_summary(job_name, measures)
new_summary_page = SUBPAGE_TMPL_CSS + SUMMARY_PAGE_TMPL_HTML.format(
id_current=id_current,
@@ -27,13 +37,15 @@ def create_summary_page(
with open(html_out, "w") as f:
f.write(new_summary_page)
+
def histogram_summary(
- job_name:str,
- ):
+ job_name: str,
+ measures: List[str],
+):
images = "
"
- for m in MEASURES:
+ for m in measures:
images += (
- f"{m} summary {job_name}
\n"
+ f"{title[m]}
\n"
+ " ".join(
[f"
" for x in FORMATS]
)
@@ -41,15 +53,26 @@ def histogram_summary(
)
return images
+
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("html_out")
parser.add_argument("id_current", type=int)
parser.add_argument("job_name")
+ parser.add_argument(
+ "--measures",
+ nargs="+",
+ help=f"List of measures to include in summary. Allowed values: {' '.join(title.keys())}",
+ default=["MLD", "DIFF", "SSNR", "ODG"],
+ )
args = parser.parse_args()
+ if not all([m in title for m in args.measures]):
+ raise ValueError(f"Invalid list of measures: {args.measures}, expected one of {' '.join(title.keys())}")
+
create_summary_page(
args.html_out,
args.id_current,
args.job_name,
+ args.measures,
)
diff --git a/ci/smoke_test.sh b/ci/smoke_test.sh
index 071b9a0be620c8ebf8aef7c00f50914459088047..32e00dfba176e4d25be81e4ccc7e00b7585599e8 100755
--- a/ci/smoke_test.sh
+++ b/ci/smoke_test.sh
@@ -28,6 +28,8 @@
# accordance with the laws of the Federal Republic of Germany excluding its conflict of law rules and
# the United Nations Convention on Contracts on the International Sales of Goods.
+set -euxo pipefail
+
function usage {
echo
echo "Usage:"
@@ -42,9 +44,9 @@ if [ ! -d "lib_com" ]; then
exit 1
fi
-if [ -z "$1" ] || [ "$1" == "test" ]; then
+if [ -z "${1:-}" ] || [ "${1:-}" == "test" ]; then
BUILD=1
-elif [ "$1" == "coverage" ]; then
+elif [ "${1:-}" == "coverage" ]; then
BUILD=0
else
usage
@@ -76,37 +78,38 @@ if [ $BUILD -eq 1 ];then
fi
# prepare combined format test signals
-echo "\n======================= 0. preparing combined format test inputs =======================\n\n"
+echo -e "\n======================= 0. preparing combined format test inputs =======================\n\n"
./scripts/prepare_combined_format_inputs.py
# run all modes vanilla-fashion
# treat ISM modes separately because passing the metadata files to MASA modes causes crashes
-ism_modes=$(./scripts/runIvasCodec.py -l | grep ISM)
-non_ism_modes=$(./scripts/runIvasCodec.py -l | grep -v ISM)
-echo "\n======================= 1. non-ism modes no FEC =======================\n\n"
-./scripts/runIvasCodec.py $verbosity_cmd -m $non_ism_modes -p $cfg $duration_arg $timeout_cmd | tee smoke_test_output.txt
-echo "\n======================= 2. ism modes no FEC =======================\n\n"
-./scripts/runIvasCodec.py $verbosity_cmd -m $ism_modes -p $cfg $duration_arg $ism_md_cmd $timeout_cmd | tee smoke_test_output.txt
+readarray -t ism_modes < <(./scripts/runIvasCodec.py -l | grep ISM)
+readarray -t non_ism_modes < <(./scripts/runIvasCodec.py -l | grep -v ISM)
+
+echo -e "\n======================= 1. non-ism modes no FEC =======================\n\n"
+./scripts/runIvasCodec.py $verbosity_cmd -m "${non_ism_modes[@]}" -p $cfg $duration_arg $timeout_cmd | tee smoke_test_output.txt
+echo -e "\n======================= 2. ism modes no FEC =======================\n\n"
+./scripts/runIvasCodec.py $verbosity_cmd -m "${ism_modes[@]}" -p $cfg $duration_arg $ism_md_cmd $timeout_cmd | tee smoke_test_output.txt
# all modes with simulated network delay - this includes JBM TSM and lost frames
-echo "\n======================= 3. JBM =======================\n\n"
+echo -e "\n======================= 3. JBM =======================\n\n"
./scripts/runIvasCodec.py $verbosity_cmd -p $cfg $duration_arg --decoder_only --jbm_file $dly_profile $timeout_cmd | tee smoke_test_output_jbm.txt
# run all modes with binaural output using external files
-formats_with_bin_out=$(./scripts/runIvasCodec.py -L | grep -v "mono\|tereo")
-bin_out_modes="BINAURAL BINAURAL_ROOM_IR BINAURAL_ROOM_REVERB"
+readarray -t formats_with_bin_out < <(./scripts/runIvasCodec.py -L | grep -v "mono\|tereo")
+bin_out_modes=(BINAURAL BINAURAL_ROOM_IR BINAURAL_ROOM_REVERB)
-echo "\n======================= 4. binaural out with HRTF files - WB =======================\n\n"
-wb_modes=$(./scripts/runIvasCodec.py -l -C $formats_with_bin_out | grep _wb_)
+echo -e "\n======================= 4. binaural out with HRTF files - WB =======================\n\n"
+readarray -t wb_modes < <(./scripts/runIvasCodec.py -l -C "${formats_with_bin_out[@]}" | grep _wb_)
hrtf_wb="../scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_16kHz.bin"
-./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m $wb_modes $duration_arg -D="-hrtf ${hrtf_wb}" --decoder_only --oc $bin_out_modes $timeout_cmd | tee -a smoke_test_output_hrtf.txt
+./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m "${wb_modes[@]}" $duration_arg -D="-hrtf ${hrtf_wb}" --decoder_only --oc "${bin_out_modes[@]}" $timeout_cmd | tee -a smoke_test_output_hrtf.txt
-echo "\n======================= 5. binaural out with HRTF files - SWB =======================\n\n"
-swb_modes=$(./scripts/runIvasCodec.py -l -C $formats_with_bin_out | grep _swb_)
+echo -e "\n======================= 5. binaural out with HRTF files - SWB =======================\n\n"
+readarray -t swb_modes < <(./scripts/runIvasCodec.py -l -C "${formats_with_bin_out[@]}" | grep _swb_)
hrtf_swb="../scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_32kHz.bin"
-./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m $swb_modes $duration_arg -D="-hrtf ${hrtf_swb}" --decoder_only --oc $bin_out_modes $timeout_cmd | tee -a smoke_test_output_hrtf.txt
+./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m "${swb_modes[@]}" $duration_arg -D="-hrtf ${hrtf_swb}" --decoder_only --oc "${bin_out_modes[@]}" $timeout_cmd | tee -a smoke_test_output_hrtf.txt
-echo "\n======================= 6. binaural out with HRTF files - FB =======================\n\n"
-fb_modes=$(./scripts/runIvasCodec.py -l -C $formats_with_bin_out | grep _fb_)
+echo -e "\n======================= 6. binaural out with HRTF files - FB =======================\n\n"
+readarray -t fb_modes < <(./scripts/runIvasCodec.py -l -C "${formats_with_bin_out[@]}" | grep _fb_)
hrtf_fb="../scripts/binauralRenderer_interface/binaural_renderers_hrtf_data/ivas_binaural_48kHz.bin"
-./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m $fb_modes $duration_arg -D="-hrtf ${hrtf_fb}" --decoder_only --oc $bin_out_modes $timeout_cmd | tee -a smoke_test_output_hrtf.txt
+./scripts/runIvasCodec.py $verbosity_cmd -p $cfg -m "${fb_modes[@]}" $duration_arg -D="-hrtf ${hrtf_fb}" --decoder_only --oc "${bin_out_modes[@]}" $timeout_cmd | tee -a smoke_test_output_hrtf.txt
diff --git a/scripts/pyaudio3dtools/audiofile.py b/scripts/pyaudio3dtools/audiofile.py
index bbee88ca9c52f570489858bf5c779c5ed4a06984..a0af594f40af0376970d18f6a034a7f0c7dba02e 100644
--- a/scripts/pyaudio3dtools/audiofile.py
+++ b/scripts/pyaudio3dtools/audiofile.py
@@ -676,9 +676,7 @@ def get_wav_file_info(filename: str) -> dict:
"""
- fid = open(filename, "rb")
-
- try:
+ with open(filename, "rb") as fid:
riff = fid.read(4)
if riff == b"RIFF":
@@ -686,13 +684,13 @@ def get_wav_file_info(filename: str) -> dict:
elif riff == b"RIFX":
binary_format = ">"
else:
- raise ValueError("No RIFF!")
+ raise ValueError(f"Not a RIFF file: {filename}")
wav_size = struct.unpack(f"{binary_format}I", fid.read(4))[0]
wav_identifier = fid.read(4)
if wav_identifier != b"WAVE":
- raise ValueError("No WAVE!")
+ raise ValueError(f"Not a WAVE file: {filename}")
fmt_chunk_id = fid.read(4)
@@ -715,9 +713,6 @@ def get_wav_file_info(filename: str) -> dict:
else:
raise ValueError("No or corrupt fmt chunk!")
- finally:
- fid.close()
-
return {
"size": wav_size,
"format_tag": wav_format,
diff --git a/scripts/pyivastest/IvasModeAnalyzer.py b/scripts/pyivastest/IvasModeAnalyzer.py
index 91bf0b44ae92cac34e8fdcc13313885628df155f..e73d7fbcb5403cc0b866db62e24f37f3dbc5c3ab 100644
--- a/scripts/pyivastest/IvasModeAnalyzer.py
+++ b/scripts/pyivastest/IvasModeAnalyzer.py
@@ -296,9 +296,8 @@ class IvasModeAnalyzer(IvasModeCollector):
keywords = [keywords]
if os.path.exists(filename):
- fp = open(filename)
- loglines = fp.readlines()
- fp.close()
+ with open(filename) as fp:
+ loglines = fp.readlines()
matching_lines = [
line
for line in loglines
@@ -571,8 +570,8 @@ class IvasModeAnalyzer(IvasModeCollector):
"""
loglines = []
if os.path.exists(filename):
- fp = open(filename)
- loglines = fp.readlines()
+ with open(filename) as fp:
+ loglines = fp.readlines()
vg_errors = {"errors": 0, "contexts": 0, "heap_bytes": 0, "heap_blocks": 0}
for line in loglines:
# print(line)
@@ -602,8 +601,8 @@ class IvasModeAnalyzer(IvasModeCollector):
"""
loglines = []
if os.path.exists(filename):
- fp = open(filename)
- loglines = fp.readlines()
+ with open(filename) as fp:
+ loglines = fp.readlines()
build_errors = 0
for line in loglines:
# print(line)
@@ -1053,8 +1052,8 @@ class IvasModeAnalyzer(IvasModeCollector):
def get_snr_report(file_name):
SNR_report = {}
if os.path.exists(file_name):
- fp = open(file_name, "r")
- lines = fp.readlines()
+ with open(file_name) as fp:
+ lines = fp.readlines()
for line in lines:
line = line.rstrip("\n")
if re.findall("SNR report", line):
diff --git a/scripts/pyivastest/IvasModeRunner.py b/scripts/pyivastest/IvasModeRunner.py
index 41dd4d20ab0f296d72b204c6482a50c955e689ed..546cc2e82cbd16b727264c611eaaadd7f799bfdf 100644
--- a/scripts/pyivastest/IvasModeRunner.py
+++ b/scripts/pyivastest/IvasModeRunner.py
@@ -242,15 +242,14 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
"""
- self.lock.acquire()
- if self.stats:
- if not self.run_encoder:
- if config["config"]["num_dec_remaining"] == config["config"]["num_dec"]:
- self.stats["num_modes_running"] += 1
- self.stats["num_decs_running"] += 1
- config["config"]["num_dec_remaining"] -= 1
- self.show_progress()
- self.lock.release()
+ with self.lock:
+ if self.stats:
+ if not self.run_encoder:
+ if config["config"]["num_dec_remaining"] == config["config"]["num_dec"]:
+ self.stats["num_modes_running"] += 1
+ self.stats["num_decs_running"] += 1
+ config["config"]["num_dec_remaining"] -= 1
+ self.show_progress()
enc_dec_cmd = config["config"]["cmd"]
item_base_name = os.path.splitext(os.path.basename(config["enc_file_name"]))[0]
output_config = config["out_config"]
@@ -261,147 +260,144 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
dec_log_name = self.get_dec_log_file_name(
self.dir_name, item_base_name, output_config
)
- dec_log = open(dec_log_name, "w")
- if config["enc_returncode"] == 0:
- dec_file_name = self.get_dec_file_name(
- self.dir_name, item_base_name, output_config, out_ext
- )
- if self.sample_rate_dec_out:
- out_sr = self.sample_rate_dec_out
- else:
- if (
- isinstance(config["dec_options"], dict)
- and "fs_out" in config["dec_options"]
- ):
- out_sr = config["dec_options"]["fs_out"]
- else:
- out_sr = BW_TO_SR[config["config"]["cmd"]["bw"]]
- dec_options = []
- if isinstance(config["dec_options"], dict):
- if "dec_options" in config["dec_options"]:
- dec_options.extend(config["dec_options"]["dec_options"])
- else:
- dec_options.extend(config["dec_options"])
- dec_options.extend(self.decoder_cmdline_options)
-
- if self.multiple_res_dir is True:
- dec_options.extend(["-info", os.path.basename(dec_file_name)])
-
- # handle SOFA file passing for group B binaural renderer
- try:
- br_kbps = int(config["config"]["cmd"]["bitrate"]) // 1000
- except ValueError:
- # this handles the case when a bitrate switching file is given
- # for now do nothing here
- br_kbps = -1
- except Exception as e:
- self.logger.error("Exception in ivas_dec_thread(): " + str(e))
- return
-
- if not config["mono"]:
- dec_options.extend([output_config])
- dec_options = [
- x.format(dec_file_name=dec_file_name) if "{dec_file_name}" in x else x
- for x in dec_options
- if x != []
- ]
- enc_file_name = config["enc_file_name"]
- self.logger.info("Decoding {} to {}".format(enc_file_name, dec_file_name))
- if self.test_tool != "":
- dec_cmd = (
- self.test_tool
- + [self.decoder]
- + dec_options
- + [str(out_sr)]
- + [enc_file_name, dec_file_name]
- )
- else:
- dec_cmd = (
- [self.decoder]
- + dec_options
- + [str(out_sr)]
- + [enc_file_name, dec_file_name]
+ with open(dec_log_name, "w") as dec_log:
+ if config["enc_returncode"] == 0:
+ dec_file_name = self.get_dec_file_name(
+ self.dir_name, item_base_name, output_config, out_ext
)
+ if self.sample_rate_dec_out:
+ out_sr = self.sample_rate_dec_out
+ else:
+ if (
+ isinstance(config["dec_options"], dict)
+ and "fs_out" in config["dec_options"]
+ ):
+ out_sr = config["dec_options"]["fs_out"]
+ else:
+ out_sr = BW_TO_SR[config["config"]["cmd"]["bw"]]
+ dec_options = []
+ if isinstance(config["dec_options"], dict):
+ if "dec_options" in config["dec_options"]:
+ dec_options.extend(config["dec_options"]["dec_options"])
+ else:
+ dec_options.extend(config["dec_options"])
+ dec_options.extend(self.decoder_cmdline_options)
+
+ if self.multiple_res_dir is True:
+ dec_options.extend(["-info", os.path.basename(dec_file_name)])
+
+ # handle SOFA file passing for group B binaural renderer
+ try:
+ br_kbps = int(config["config"]["cmd"]["bitrate"]) // 1000
+ except ValueError:
+ # this handles the case when a bitrate switching file is given
+ # for now do nothing here
+ br_kbps = -1
+ except Exception as e:
+ self.logger.error("Exception in ivas_dec_thread(): " + str(e))
+ return
+
+ if not config["mono"]:
+ dec_options.extend([output_config])
+ dec_options = [
+ x.format(dec_file_name=dec_file_name) if "{dec_file_name}" in x else x
+ for x in dec_options
+ if x != []
+ ]
+ enc_file_name = config["enc_file_name"]
+ self.logger.info("Decoding {} to {}".format(enc_file_name, dec_file_name))
+ if self.test_tool != "":
+ dec_cmd = (
+ self.test_tool
+ + [self.decoder]
+ + dec_options
+ + [str(out_sr)]
+ + [enc_file_name, dec_file_name]
+ )
+ else:
+ dec_cmd = (
+ [self.decoder]
+ + dec_options
+ + [str(out_sr)]
+ + [enc_file_name, dec_file_name]
+ )
- dec_log.write(" ".join(dec_cmd))
- try:
- cur_dec_result = subprocess.run(
- dec_cmd,
- capture_output=True,
- text=True,
- env=self.run_env,
- timeout=self.timeout,
- )
- returncode = cur_dec_result.returncode
- dec_log.write(cur_dec_result.stderr)
- dec_log.write(cur_dec_result.stdout)
- except subprocess.TimeoutExpired:
- returncode = RET_CODE_TIMEOUT_EXP
-
- if returncode != 0:
- fail_string = "Decoding {} to {} failed!"
- if returncode == RET_CODE_TIMEOUT_EXP:
- fail_string = (
- fail_string[:-1]
- + f" due to timeout after {self.timeout} seconds!"
+ dec_log.write(" ".join(dec_cmd))
+ try:
+ cur_dec_result = subprocess.run(
+ dec_cmd,
+ capture_output=True,
+ text=True,
+ env=self.run_env,
+ timeout=self.timeout,
)
+ returncode = cur_dec_result.returncode
+ dec_log.write(cur_dec_result.stderr)
+ dec_log.write(cur_dec_result.stdout)
+ except subprocess.TimeoutExpired:
+ returncode = RET_CODE_TIMEOUT_EXP
+
+ if returncode != 0:
+ fail_string = "Decoding {} to {} failed "
+ if returncode == RET_CODE_TIMEOUT_EXP:
+ fail_string += f"due to timeout after {self.timeout} seconds"
+ else:
+ fail_string += f"with exit code {returncode}"
+ fail_string += "!"
- self.logger.error(fail_string.format(enc_file_name, dec_file_name))
- self.lock.acquire()
- if self.stats:
- self.stats["num_dec_errors"] += 1
- self.show_progress()
- self.results.append(
- [
- fail_string.format(output_config, enc_file_name),
- config["ivas_format"],
- enc_dec_cmd["table_name"],
- dec_log_name,
- ]
- )
+ self.logger.error(fail_string.format(enc_file_name, dec_file_name))
+ with self.lock:
+ if self.stats:
+ self.stats["num_dec_errors"] += 1
+ self.show_progress()
+ self.results.append(
+ [
+ fail_string.format(output_config, enc_file_name),
+ config["ivas_format"],
+ enc_dec_cmd["table_name"],
+ dec_log_name,
+ ]
+ )
- # get mode from enc_file_name
- file_base_no_ext = os.path.basename(enc_file_name)[
- :-4
- ] # NOTE: this assumes that all encoded files still end in ".192"
- in_config = config["config"]["cmd"]["in_config"]
- mode = file_base_no_ext[file_base_no_ext.rfind(in_config) :]
- if output_config in self.failed_modes["dec"]:
- self.failed_modes["dec"][output_config].append(mode)
+ # get mode from enc_file_name
+ file_base_no_ext = os.path.basename(enc_file_name)[
+ :-4
+ ] # NOTE: this assumes that all encoded files still end in ".192"
+ in_config = config["config"]["cmd"]["in_config"]
+ mode = file_base_no_ext[file_base_no_ext.rfind(in_config) :]
+ if output_config in self.failed_modes["dec"]:
+ self.failed_modes["dec"][output_config].append(mode)
+ else:
+ self.failed_modes["dec"][output_config] = [mode]
else:
- self.failed_modes["dec"][output_config] = [mode]
- self.lock.release()
+ self.logger.info(
+ "Decoding successful for {} to {}".format(
+ enc_file_name, dec_file_name
+ )
+ )
else:
- self.logger.info(
- "Decoding successful for {} to {}".format(
- enc_file_name, dec_file_name
+ self.logger.error(
+ "No decoding of {} to {}, encoding failed already".format(
+ config["enc_file_name"], output_config
)
)
- else:
- self.logger.error(
- "No decoding of {} to {}, encoding failed already".format(
- config["enc_file_name"], output_config
- )
- )
- dec_log.write(
- "No decoding of {} to {}, encoding failed already".format(
- config["enc_file_name"], output_config
+ dec_log.write(
+ "No decoding of {} to {}, encoding failed already".format(
+ config["enc_file_name"], output_config
+ )
)
- )
- self.lock.acquire()
-
- config["config"]["num_dec_done"] += 1
- if self.stats:
- self.stats["num_decs_finished"] += 1
- self.stats["num_decs_running"] -= 1
- if config["enc_returncode"] != 0:
- self.stats["num_dec_errors"] += 1
- if config["config"]["num_dec_done"] == config["config"]["num_dec"]:
- self.stats["num_modes_running"] -= 1
- self.stats["num_modes_finished"] += 1
- self.show_progress()
- self.lock.release()
+ with self.lock:
+ config["config"]["num_dec_done"] += 1
+ if self.stats:
+ self.stats["num_decs_finished"] += 1
+ self.stats["num_decs_running"] -= 1
+ if config["enc_returncode"] != 0:
+ self.stats["num_dec_errors"] += 1
+ if config["config"]["num_dec_done"] == config["config"]["num_dec"]:
+ self.stats["num_modes_running"] -= 1
+ self.stats["num_modes_finished"] += 1
+ self.show_progress()
def clean_pcm_directory(self):
"""
@@ -439,33 +435,30 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
error = 0
enc_dec_cmd = deepcopy(config["cmd"])
# get next item
- config["lock"].acquire()
- self.lock.acquire()
- if self.stats:
- if config["num_enc_remaining"] == config["num_enc"]:
- self.stats["num_modes_running"] += 1
+ with config["lock"]:
+ with self.lock:
+ if self.stats:
+ if config["num_enc_remaining"] == config["num_enc"]:
+ self.stats["num_modes_running"] += 1
+ self.show_progress()
+ if config["num_enc_remaining"]:
+ in_file_name = config["item_list"].pop()
+ config["num_enc_remaining"] -= 1
+ else:
+ return
+
+ metadata_file_names = list()
+
+ # TODO: is this still needed - what case does this cover?
+ if isinstance(in_file_name, list):
+ metadata_file_names = in_file_name[1:]
+ in_file_name = in_file_name[0]
+ self.logger.info("Encoding Mode {} input file {}".format(mode, in_file_name))
+
+ with self.lock:
+ if self.stats:
+ self.stats["num_encs_running"] += 1
self.show_progress()
- if config["num_enc_remaining"]:
- in_file_name = config["item_list"].pop()
- config["num_enc_remaining"] -= 1
- else:
- config["lock"].release()
- return
- self.lock.release()
-
- metadata_file_names = list()
-
- # TODO: is this still needed - what case does this cover?
- if isinstance(in_file_name, list):
- metadata_file_names = in_file_name[1:]
- in_file_name = in_file_name[0]
- self.logger.info("Encoding Mode {} input file {}".format(mode, in_file_name))
- config["lock"].release()
- self.lock.acquire()
- if self.stats:
- self.stats["num_encs_running"] += 1
- self.show_progress()
- self.lock.release()
pcm_name_lock = None
item_base_name = os.path.splitext(os.path.basename(in_file_name))[0]
enc_file_name = os.path.join(
@@ -524,28 +517,35 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
pcm_name = os.path.join(self.dir_name, "pcm", pcm_base_name)
pcm_name_lock = "".join([pcm_name, ".LOCK"])
pcm_info_file = f"{pcm_name}.json"
- self.lock.acquire()
-
- if not os.path.exists(pcm_name_lock):
- if not os.path.exists(pcm_name):
- # create the lock
- fp = open(pcm_name_lock, "w")
- fp.close()
- self.lock.release()
- self.logger.info(
- "Creating input pcm for item {} at sample rate {}: {}".format(
- item_base_name, sample_rate_in, pcm_name
- )
- )
- pcm_log_name_tmp = "".join(
- [pcm_base_name, constants.LOG_FILE_EXT]
- )
- pcm_log_name = os.path.join(
- self.dir_name, "logs", pcm_log_name_tmp
+ create_input_file = False
+ pcm_ready = False
+
+ with self.lock:
+ if not os.path.exists(pcm_name_lock):
+ if not os.path.exists(pcm_name):
+ # create lock file
+ with open(pcm_name_lock, "w"):
+ pass
+
+ create_input_file = True
+
+ if create_input_file:
+
+ self.logger.info(
+ "Creating input pcm for item {} at sample rate {}: {}".format(
+ item_base_name, sample_rate_in, pcm_name
)
- pcm_log = open(pcm_log_name, "w")
+ )
+ pcm_log_name_tmp = "".join(
+ [pcm_base_name, constants.LOG_FILE_EXT]
+ )
+ pcm_log_name = os.path.join(
+ self.dir_name, "logs", pcm_log_name_tmp
+ )
+
+ with open(pcm_log_name, "w") as pcm_log:
pcm_name_res_tmp = pcm_name + ".res.wav"
pcm_name_cpy_tmp = pcm_name + ".cpy.wav"
in_file_name_transformed = self.transform_path(in_file_name)
@@ -594,8 +594,8 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
cut_len_samples = end_time_samples - start_time_samples
if (
- cut_len_samples + start_time_samples < in_len
- or start_time_samples > 0
+ cut_len_samples + start_time_samples < in_len
+ or start_time_samples > 0
):
end_time_samples = min(end_time_samples, in_len)
sig = ar.cut(
@@ -623,50 +623,45 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
in_config = config["cmd"]["in_config"].upper()
- pcm_log.flush()
- pcm_log.close()
# remove file lock
- self.lock.acquire()
- os.remove(pcm_name_lock)
- # os.remove(pcm_name_res_tmp)
- if do_limit_duration and cut_len_samples < in_len:
- os.remove(pcm_name_cpy_tmp)
- self.logger.info(
- "PCM file {} successfully created!".format(pcm_name)
- )
- self.lock.release()
- else:
- self.lock.release()
- else:
- # wait for pcm to be ready
- self.lock.release()
- not_ready = 1
- while not_ready:
- time.sleep(1)
- self.lock.acquire()
+ with self.lock:
+ os.remove(pcm_name_lock)
+ # os.remove(pcm_name_res_tmp)
+ if do_limit_duration and cut_len_samples < in_len:
+ os.remove(pcm_name_cpy_tmp)
+ self.logger.info(
+ "PCM file {} successfully created!".format(pcm_name)
+ )
+
+ # PCM is now ready, no need to wait for anybody
+ pcm_ready = True
+
+ # wait for pcm to be ready
+ while not pcm_ready:
+ time.sleep(1)
+
+ with self.lock:
if not os.path.exists(pcm_name_lock):
- not_ready = 0
- self.lock.release()
+ pcm_ready = True
else:
pcm_name = in_file_name
pcm_info_file = f"{pcm_name}.json"
# get input format dictionary for the input file
- self.lock.acquire()
- if pcm_name not in self.pcm_info:
- if os.path.exists(pcm_info_file):
- with open(pcm_info_file, "r") as pcmifp:
- in_format_dict = json.load(pcmifp)
+ with self.lock:
+ if pcm_name not in self.pcm_info:
+ if os.path.exists(pcm_info_file):
+ with open(pcm_info_file, "r") as pcmifp:
+ in_format_dict = json.load(pcmifp)
- else:
- # add in_format_dict according to the in_format of the mode...
- if in_format == "SBA":
- # use HOA3 as default for SBA if nothing more is known...
- in_format = "HOA3"
- in_format_dict = spformat.get_format_dict(in_format)
- self.pcm_info.update({pcm_name: in_format_dict})
- self.lock.release()
+ else:
+ # add in_format_dict according to the in_format of the mode...
+ if in_format == "SBA":
+ # use HOA3 as default for SBA if nothing more is known...
+ in_format = "HOA3"
+ in_format_dict = spformat.get_format_dict(in_format)
+ self.pcm_info.update({pcm_name: in_format_dict})
# build the encoder commandline
enc_options = enc_dec_cmd["encmodeoption"]
@@ -754,128 +749,124 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
for x in enc_cmd
]
- enc_log = open(enc_log_name, "w")
- enc_log.write(" ".join(enc_cmd))
- try:
- enc_result = subprocess.run(
- enc_cmd,
- capture_output=True,
- text=True,
- env=self.run_env,
- timeout=self.timeout,
- )
- error = enc_result.returncode
- enc_log.write(enc_result.stderr)
- enc_log.write(enc_result.stdout)
- except subprocess.TimeoutExpired:
- error = RET_CODE_TIMEOUT_EXP
-
- if error == 0 and "bitstream_processing" in enc_dec_cmd:
- bs_in_file = enc_file_name
- proc_chain = deepcopy(enc_dec_cmd["bitstream_processing"]["proc_chain"])
- for processing in proc_chain:
- suffix = processing.pop()
- bs_out_file = ".".join(
- ["_".join([os.path.splitext(bs_in_file)[0], suffix]), "192"]
- )
- # process
- proc_cmd = deepcopy(processing)
- proc_cmd = [
- x.format(in_file=bs_in_file) if "{in_file}" in x else x
- for x in proc_cmd
- ]
- proc_cmd = [
- x.format(out_file=bs_out_file) if "{out_file}" in x else x
- for x in proc_cmd
- ]
- enc_log = open(enc_log_name, "a")
- enc_log.write(" ".join(proc_cmd))
- proc_result = subprocess.run(
- proc_cmd, capture_output=True, text=True, encoding="utf8"
+ with open(enc_log_name, "w") as enc_log:
+ enc_log.write(" ".join(enc_cmd))
+ try:
+ enc_result = subprocess.run(
+ enc_cmd,
+ capture_output=True,
+ text=True,
+ env=self.run_env,
+ timeout=self.timeout,
)
- enc_log.write(proc_result.stderr)
- enc_log.write(proc_result.stdout)
- error = proc_result.returncode
- if error:
- self.logger.error(
- "Processing step {} for {} failed!".format(
- suffix, enc_file_name
- )
+ error = enc_result.returncode
+ enc_log.write(enc_result.stderr)
+ enc_log.write(enc_result.stdout)
+ except subprocess.TimeoutExpired:
+ error = RET_CODE_TIMEOUT_EXP
+
+ if error == 0 and "bitstream_processing" in enc_dec_cmd:
+ bs_in_file = enc_file_name
+ proc_chain = deepcopy(enc_dec_cmd["bitstream_processing"]["proc_chain"])
+ for processing in proc_chain:
+ suffix = processing.pop()
+ bs_out_file = ".".join(
+ ["_".join([os.path.splitext(bs_in_file)[0], suffix]), "192"]
)
- self.logger.error(proc_result.stderr)
- raise RuntimeError(
- "Processing step {} for {} failed!\n{}".format(
- suffix, enc_file_name, proc_result.stderr
- )
+ # process
+ proc_cmd = deepcopy(processing)
+ proc_cmd = [
+ x.format(in_file=bs_in_file) if "{in_file}" in x else x
+ for x in proc_cmd
+ ]
+ proc_cmd = [
+ x.format(out_file=bs_out_file) if "{out_file}" in x else x
+ for x in proc_cmd
+ ]
+ enc_log.write(" ".join(proc_cmd))
+ proc_result = subprocess.run(
+ proc_cmd, capture_output=True, text=True, encoding="utf8"
)
- bs_in_file = bs_out_file
- enc_file_name_dec = bs_out_file
- else:
- enc_file_name_dec = enc_file_name
+ enc_log.write(proc_result.stderr)
+ enc_log.write(proc_result.stdout)
+ error = proc_result.returncode
+ if error:
+ self.logger.error(
+ "Processing step {} for {} failed!".format(
+ suffix, enc_file_name
+ )
+ )
+ self.logger.error(proc_result.stderr)
+ raise RuntimeError(
+ "Processing step {} for {} failed!\n{}".format(
+ suffix, enc_file_name, proc_result.stderr
+ )
+ )
+ bs_in_file = bs_out_file
+ enc_file_name_dec = bs_out_file
+ else:
+ enc_file_name_dec = enc_file_name
except Exception as exc:
# make sure we do not have a deadlock...
- self.lock.acquire()
- if pcm_name_lock:
-
- if os.path.exists(pcm_name_lock):
- # something went wrong with PCM creation, so make sure all waiting threads can continue...
- self.logger.error(
- "Something went wrong with creating the PCM file {}".format(
- pcm_name
+ with self.lock:
+ if pcm_name_lock:
+
+ if os.path.exists(pcm_name_lock):
+ # something went wrong with PCM creation, so make sure all waiting threads can continue...
+ self.logger.error(
+ "Something went wrong with creating the PCM file {}".format(
+ pcm_name
+ )
)
- )
- os.remove(pcm_name_lock)
- os.remove(pcm_name)
+ os.remove(pcm_name_lock)
+ os.remove(pcm_name)
- self.logger.console(
- "Exception when encoding item {}: {}".format(enc_file_name, str(exc)),
- logging.ERROR,
- )
- self.logger.console(
- "Traceback: {}".format(traceback.format_tb(exc.__traceback__)),
- logging.ERROR,
- )
- self.lock.release()
+ self.logger.console(
+ "Exception when encoding item {}: {}".format(enc_file_name, str(exc)),
+ logging.ERROR,
+ )
+ self.logger.console(
+ "Traceback: {}".format(traceback.format_tb(exc.__traceback__)),
+ logging.ERROR,
+ )
enc_file_name_dec = enc_file_name
error = 1
- self.lock.acquire()
- if self.stats:
- config["num_enc_done"] += 1
- if config["num_enc_done"] == config["num_enc"]:
- self.enc_queue["num_modes_enc_done"] += 1
- if not self.run_decoder:
- self.stats["num_modes_running"] -= 1
- self.stats["num_modes_finished"] -= 1
- if self.enc_queue["num_modes_enc"] == self.enc_queue["num_modes_enc_done"]:
- self.dec_queue["all_encoded"] = True
- self.stats["num_encs_finished"] += 1
- self.stats["num_encs_running"] -= 1
- self.show_progress()
- self.lock.release()
+ with self.lock:
+ if self.stats:
+ config["num_enc_done"] += 1
+ if config["num_enc_done"] == config["num_enc"]:
+ self.enc_queue["num_modes_enc_done"] += 1
+ if not self.run_decoder:
+ self.stats["num_modes_running"] -= 1
+ self.stats["num_modes_finished"] -= 1
+ if self.enc_queue["num_modes_enc"] == self.enc_queue["num_modes_enc_done"]:
+ self.dec_queue["all_encoded"] = True
+ self.stats["num_encs_finished"] += 1
+ self.stats["num_encs_running"] -= 1
+ self.show_progress()
if error != 0:
- fail_string = "Encoding failed for {}"
+ fail_string = "error " + str(error) + ": Encoding failed for {}"
if error == RET_CODE_TIMEOUT_EXP:
fail_string = (
fail_string + f" due to timeout after {self.timeout} seconds"
)
- self.lock.acquire()
- if self.stats:
- self.stats["num_enc_errors"] += 1
- self.show_progress()
- self.results.append(
- [
- fail_string.format(enc_file_name),
- mode,
- enc_dec_cmd["table_name"],
- enc_log_name,
- ]
- )
- self.failed_modes["enc"].append(mode)
- self.lock.release()
+ with self.lock:
+ if self.stats:
+ self.stats["num_enc_errors"] += 1
+ self.show_progress()
+ self.results.append(
+ [
+ fail_string.format(enc_file_name),
+ mode,
+ enc_dec_cmd["table_name"],
+ enc_log_name,
+ ]
+ )
+ self.failed_modes["enc"].append(mode)
self.logger.error(fail_string.format(enc_file_name))
else:
self.logger.info("Encoding successful for {}".format(enc_file_name))
@@ -1169,16 +1160,15 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
return missing_mode_flat_dict
def bs_processing_thread(self, bs_entry):
- self.lock.acquire()
- self.bs_processing_queue["stats"]["num_bs_running"] += 1
- line = "Bit stream processing: {}/{} ({} running)".format(
- self.bs_processing_queue["stats"]["num_bs_done"],
- self.bs_processing_queue["stats"]["num_bs_total"],
- self.bs_processing_queue["stats"]["num_bs_running"],
- )
- self.logger.progress(line)
- enc_file_name = bs_entry["item_list"].pop()
- self.lock.release()
+ with self.lock:
+ self.bs_processing_queue["stats"]["num_bs_running"] += 1
+ line = "Bit stream processing: {}/{} ({} running)".format(
+ self.bs_processing_queue["stats"]["num_bs_done"],
+ self.bs_processing_queue["stats"]["num_bs_total"],
+ self.bs_processing_queue["stats"]["num_bs_running"],
+ )
+ self.logger.progress(line)
+ enc_file_name = bs_entry["item_list"].pop()
bs_in_file = enc_file_name
proc_chain = deepcopy(bs_entry["proc_chain"])
suffices = self.get_bs_processing_suffices(bs_entry)
@@ -1195,60 +1185,60 @@ class IvasModeRunner(IvasModeCollector.IvasModeCollector):
]
),
)
- enc_log = open(enc_log_name, "w")
- self.logger.info("Processing bit stream {}".format(enc_file_name))
- try:
- for processing in proc_chain:
- suffix = processing.pop()
- bs_out_file = ".".join(
- ["_".join([os.path.splitext(bs_in_file)[0], suffix]), "192"]
- )
- # process
- proc_cmd = deepcopy(processing)
- proc_cmd = [
- x.format(in_file=bs_in_file) if "{in_file}" in x else x
- for x in proc_cmd
- ]
- proc_cmd = [
- x.format(out_file=bs_out_file) if "{out_file}" in x else x
- for x in proc_cmd
- ]
- enc_log.write(" ".join(proc_cmd))
- proc_result = subprocess.run(proc_cmd, capture_output=True, text=True)
- enc_log.write(proc_result.stderr)
- enc_log.write(proc_result.stdout)
- error = proc_result.returncode
- if error:
- self.logger.error(
- "Processing step {} for {} failed!".format(
- suffix, enc_file_name
- )
+ with open(enc_log_name, "w") as enc_log:
+ self.logger.info("Processing bit stream {}".format(enc_file_name))
+ try:
+ for processing in proc_chain:
+ suffix = processing.pop()
+ bs_out_file = ".".join(
+ ["_".join([os.path.splitext(bs_in_file)[0], suffix]), "192"]
)
- raise RuntimeError(
- "Processing step {} for {} failed!".format(
- suffix, enc_file_name
+ # process
+ proc_cmd = deepcopy(processing)
+ proc_cmd = [
+ x.format(in_file=bs_in_file) if "{in_file}" in x else x
+ for x in proc_cmd
+ ]
+ proc_cmd = [
+ x.format(out_file=bs_out_file) if "{out_file}" in x else x
+ for x in proc_cmd
+ ]
+
+ enc_log.write(" ".join(proc_cmd))
+ proc_result = subprocess.run(proc_cmd, capture_output=True, text=True)
+ enc_log.write(proc_result.stderr)
+ enc_log.write(proc_result.stdout)
+ error = proc_result.returncode
+ if error:
+ self.logger.error(
+ "Processing step {} for {} failed!".format(
+ suffix, enc_file_name
+ )
)
- )
- bs_in_file = bs_out_file
- except Exception as exc:
- self.logger.error(
- "Exception processing bitstream {}: {}".format(enc_file_name, str(exc))
- )
- self.logger.error("failed command: {}".format(" ".join(proc_cmd)))
- self.logger.error(
- "Traceback: {}".format(traceback.format_tb(exc.__traceback__))
- )
- self.lock.acquire()
- self.bs_processing_queue["stats"]["num_bs_running"] -= 1
- self.bs_processing_queue["stats"]["num_bs_done"] += 1
- line = "Bit stream processing: {}/{} ({} running)".format(
- self.bs_processing_queue["stats"]["num_bs_done"],
- self.bs_processing_queue["stats"]["num_bs_total"],
- self.bs_processing_queue["stats"]["num_bs_running"],
- )
- self.logger.progress(line)
- self.lock.release()
+ raise RuntimeError(
+ "Processing step {} for {} failed!".format(
+ suffix, enc_file_name
+ )
+ )
+ bs_in_file = bs_out_file
+ except Exception as exc:
+ self.logger.error(
+ "Exception processing bitstream {}: {}".format(enc_file_name, str(exc))
+ )
+ self.logger.error("failed command: {}".format(" ".join(proc_cmd)))
+ self.logger.error(
+ "Traceback: {}".format(traceback.format_tb(exc.__traceback__))
+ )
+ with self.lock:
+ self.bs_processing_queue["stats"]["num_bs_running"] -= 1
+ self.bs_processing_queue["stats"]["num_bs_done"] += 1
+ line = "Bit stream processing: {}/{} ({} running)".format(
+ self.bs_processing_queue["stats"]["num_bs_done"],
+ self.bs_processing_queue["stats"]["num_bs_total"],
+ self.bs_processing_queue["stats"]["num_bs_running"],
+ )
+ self.logger.progress(line)
def run_bs_processing_queue(self):
self.logger.console("Postprocessing of bit streams ...", logging.INFO)