From 20f30f75881e385c4365da294688dade620e103e Mon Sep 17 00:00:00 2001 From: knj Date: Mon, 22 May 2023 17:29:20 +0200 Subject: [PATCH 1/6] prevent modifying DEFAULT_CONFIG in merge_dicts --- ivas_processing_scripts/processing/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ivas_processing_scripts/processing/config.py b/ivas_processing_scripts/processing/config.py index 1fef013b..d7b0ad6b 100755 --- a/ivas_processing_scripts/processing/config.py +++ b/ivas_processing_scripts/processing/config.py @@ -76,7 +76,7 @@ class TestConfig: self.tmp_dirs = [] # get default config - cfg = DEFAULT_CONFIG + cfg = deepcopy(DEFAULT_CONFIG) # parse configuration file file_cfg = self._parse_yaml(filename) -- GitLab From 22ef3393c4b3da4078ef5b6a9ecf1021c9f52d0f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 24 May 2023 14:44:32 +0200 Subject: [PATCH 2/6] make win binaries with wine the default on linux --- ivas_processing_scripts/constants.py | 17 +--------- ivas_processing_scripts/processing/chains.py | 2 ++ ivas_processing_scripts/processing/config.py | 35 ++++++++++++++++---- ivas_processing_scripts/processing/evs.py | 10 ++++-- ivas_processing_scripts/processing/ivas.py | 11 ++++-- 5 files changed, 48 insertions(+), 27 deletions(-) diff --git a/ivas_processing_scripts/constants.py b/ivas_processing_scripts/constants.py index 762c11b3..5c6b348a 100755 --- a/ivas_processing_scripts/constants.py +++ b/ivas_processing_scripts/constants.py @@ -58,6 +58,7 @@ DEFAULT_CONFIG = { "date": f"{datetime.now().strftime('%Y%m%d_%H.%M.%S')}", "git_sha": f"{get_gitsha()}", "multiprocessing": True, + "use_windows_codec_binaries": True, "delete_tmp": False, "master_seed": 0, "prerun_seed": 0, @@ -69,22 +70,6 @@ DEFAULT_CONFIG = { }, "condition_in_output_filename": False, } -DEFAULT_CONFIG_EVS = { - "cod": { - "bin": find_binary("EVS_cod", raise_error=False), - }, - "dec": { - "bin": find_binary("EVS_dec", raise_error=False), - }, -} -DEFAULT_CONFIG_IVAS = { - "cod": { - "bin": find_binary("IVAS_cod", raise_error=False), - }, - "dec": { - "bin": find_binary("IVAS_dec", raise_error=False), - }, -} DEFAULT_CONFIG_BINARIES = { "binary_paths": get_binary_paths( diff --git a/ivas_processing_scripts/processing/chains.py b/ivas_processing_scripts/processing/chains.py index c11be9a1..24d7e806 100755 --- a/ivas_processing_scripts/processing/chains.py +++ b/ivas_processing_scripts/processing/chains.py @@ -323,6 +323,7 @@ def get_processing_chain( "preamble": preamble, "evs_lfe_9k6bps_nb": evs_lfe_9k6bps_nb, "sba_fmt": cond_cfg.get("sba_fmt", tmp_in_fmt), + "use_windows_codec_binaries": cfg.use_windows_codec_binaries, } ) ) @@ -408,6 +409,7 @@ def get_processing_chain( "multiprocessing": cfg.multiprocessing, "tx": tx_cfg, "preamble": preamble, + "use_windows_codec_binaries": cfg.use_windows_codec_binaries, } ) ) diff --git a/ivas_processing_scripts/processing/config.py b/ivas_processing_scripts/processing/config.py index d7b0ad6b..f6a26bd0 100755 --- a/ivas_processing_scripts/processing/config.py +++ b/ivas_processing_scripts/processing/config.py @@ -34,11 +34,10 @@ from copy import deepcopy from pathlib import Path import yaml +import platform from ivas_processing_scripts.constants import ( DEFAULT_CONFIG, - DEFAULT_CONFIG_EVS, - DEFAULT_CONFIG_IVAS, REQUIRED_KEYS, REQUIRED_KEYS_ESDRU, REQUIRED_KEYS_EVS, @@ -46,7 +45,7 @@ from ivas_processing_scripts.constants import ( REQUIRED_KEYS_MNRU, SUPPORTED_CONDITIONS, ) -from ivas_processing_scripts.utils import get_abs_path +from ivas_processing_scripts.utils import get_abs_path, find_binary def merge_dicts(base: dict, other: dict) -> None: @@ -64,6 +63,21 @@ def merge_dicts(base: dict, other: dict) -> None: base[k] = other[k] +def get_default_config_for_codecs(codec_name: str, ext_with_dot: str = "") -> dict: + cod_bin = f"{codec_name}_cod{ext_with_dot}" + dec_bin = f"{codec_name}_dec{ext_with_dot}" + + cfg = { + "cod": { + "bin": find_binary(cod_bin, raise_error=False), + }, + "dec": { + "bin": find_binary(dec_bin, raise_error=False), + }, + } + return cfg + + class TestConfig: # avoid confusion with pytest tests due to naming __test__ = False @@ -82,10 +96,11 @@ class TestConfig: file_cfg = self._parse_yaml(filename) # validate configuration from file - self._validate(file_cfg) + self._validate_file_cfg(file_cfg, cfg["use_windows_codec_binaries"]) # merge dictionaries, overriding from config file merge_dicts(cfg, file_cfg) + self._validate_merged_config(cfg) # set attributes from merged dictionary self.__dict__.update(cfg) @@ -117,7 +132,7 @@ class TestConfig: return cfg - def _validate(self, cfg: dict): + def _validate_file_cfg(self, cfg: dict, use_windows_codec_binaries: bool): """ensure configuration contains required keys""" MISSING_KEYS = [] # check required keys @@ -170,6 +185,7 @@ class TestConfig: "Background noise file has to be placed outside the input folder!" ) + codec_bin_extension = ".exe" if platform.system() == "Linux" and use_windows_codec_binaries else "" for cond_name, cond_cfg in cfg.get("conditions_to_generate").items(): type = cond_cfg.get("type") if not type: @@ -181,7 +197,7 @@ class TestConfig: f"Condition type {cond_cfg.get('type')} is unsuported! Must be one of '{SUPPORTED_CONDITIONS}'" ) elif type == "evs": - merged_cfg = deepcopy(DEFAULT_CONFIG_EVS) + merged_cfg = get_default_config_for_codecs("EVS", codec_bin_extension) merge_dicts(merged_cfg, cond_cfg) cfg["conditions_to_generate"][cond_name] = merged_cfg if REQUIRED_KEYS_EVS.difference( @@ -191,7 +207,7 @@ class TestConfig: f"The following key(s) must be specified for EVS: {REQUIRED_KEYS_EVS}" ) elif type == "ivas": - merged_cfg = deepcopy(DEFAULT_CONFIG_IVAS) + merged_cfg = get_default_config_for_codecs("IVAS", codec_bin_extension) merge_dicts(merged_cfg, cond_cfg) cfg["conditions_to_generate"][cond_name] = merged_cfg if REQUIRED_KEYS_IVAS.difference( @@ -214,3 +230,8 @@ class TestConfig: raise KeyError( f"The following key must be specified for ESDRU: {REQUIRED_KEYS_ESDRU}" ) + + def _validate_merged_config(self, cfg: dict): + # if not on windows, but "use_windows_codec_binaries" is given, assure that wine is there + if platform.system() == "Linux" and cfg["use_windows_codec_binaries"] and find_binary("wine") is None: + raise FileNotFoundError("Using windows binaries on Linux requires wine") \ No newline at end of file diff --git a/ivas_processing_scripts/processing/evs.py b/ivas_processing_scripts/processing/evs.py index 0653327e..2871ac85 100755 --- a/ivas_processing_scripts/processing/evs.py +++ b/ivas_processing_scripts/processing/evs.py @@ -64,12 +64,14 @@ class EVS(Processing): self.name = "evs" self.in_fmt = audio.fromtype(self.in_fmt) self._validate() + self._use_wine = platform.system() == "Linux" and self.use_windows_codec_binaries def _validate( self, ): + need_exe_suffix = platform.system() == "Windows" or self.use_windows_codec_binaries if not self.cod_bin or not Path(self.cod_bin).exists(): - if platform.system() == "Windows" and ( + if need_exe_suffix and ( self.cod_bin and Path(self.cod_bin).with_suffix(".exe").exists() ): self.cod_bin = Path(self.cod_bin).with_suffix(".exe") @@ -78,7 +80,7 @@ class EVS(Processing): "The EVS encoder binary was not found! Please check the configuration." ) if not self.dec_bin or not Path(self.dec_bin).exists(): - if platform.system() == "Windows" and ( + if need_exe_suffix and ( self.dec_bin and Path(self.dec_bin).with_suffix(".exe").exists() ): self.dec_bin = Path(self.dec_bin).with_suffix(".exe") @@ -286,6 +288,8 @@ class EVS(Processing): logger: Optional[logging.Logger] = None, ) -> None: cmd = [self.cod_bin] + if self._use_wine: + cmd.insert(0, "wine") # in case of LFE 9.6 kbps NB processing strip any "-max_band XX" from cmd.extend and amend by "-max_band NB" if proc_chan_lfe_9k6bps_nb is True: @@ -381,6 +385,8 @@ class EVS(Processing): logger: Optional[logging.Logger] = None, ) -> None: cmd = [self.dec_bin] + if self._use_wine: + cmd.insert(0, "wine") if self.dec_opts: cmd.extend(self.dec_opts) diff --git a/ivas_processing_scripts/processing/ivas.py b/ivas_processing_scripts/processing/ivas.py index b2cae46b..b7c70c7a 100755 --- a/ivas_processing_scripts/processing/ivas.py +++ b/ivas_processing_scripts/processing/ivas.py @@ -60,10 +60,12 @@ class IVAS(Processing): self.out_fmt = audio.fromtype(self.out_fmt) if not hasattr(self, "dec_opts"): self.dec_opts = None + self._use_wine = platform.system() == "Linux" and self.use_windows_codec_binaries def _validate(self): + need_exe_suffix = platform.system() == "Windows" or self.use_windows_codec_binaries if not self.cod_bin or not Path(self.cod_bin).exists(): - if platform.system() == "Windows" and ( + if need_exe_suffix and ( self.cod_bin and Path(self.cod_bin).with_suffix(".exe").exists() ): self.cod_bin = Path(self.cod_bin).with_suffix(".exe") @@ -72,7 +74,7 @@ class IVAS(Processing): "The IVAS encoder binary was not found! Please check the configuration." ) if not self.dec_bin or not Path(self.dec_bin).exists(): - if platform.system() == "Windows" and ( + if need_exe_suffix and ( self.dec_bin and Path(self.dec_bin).with_suffix(".exe").exists() ): self.dec_bin = Path(self.dec_bin).with_suffix(".exe") @@ -181,6 +183,8 @@ class IVAS(Processing): raise ValueError(f"IVAS: invalid audio input extension: {in_file.suffix}") cmd = [self.cod_bin] + if self._use_wine: + cmd.insert(0, "wine") if self.cod_opts: cmd.extend(self.cod_opts) @@ -272,6 +276,9 @@ class IVAS(Processing): logger.debug(f"IVAS decoder {bitstream} -> {out_file}") cmd = [self.dec_bin] + if self._use_wine: + cmd.insert(0, "wine") + if hasattr(self, "trajectory"): cmd.extend(["-T", self.trajectory]) -- GitLab From a68866ba85617f5d57f60cfb2ee4f78a0ad5ce1f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 24 May 2023 14:47:30 +0200 Subject: [PATCH 3/6] add new keyto TEMPLATE --- examples/TEMPLATE.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/TEMPLATE.yml b/examples/TEMPLATE.yml index a5039d4d..9b5889ae 100755 --- a/examples/TEMPLATE.yml +++ b/examples/TEMPLATE.yml @@ -18,6 +18,9 @@ # master_seed: 5 ### Additional seed to specify number of preruns (used for background noise delay and FER bitstream processing); default = 0 # prerun_seed: 2 +### flag for linux to use windows-built binaries with wine: default = true +### this requires the wine binary to be available and will be ignored on windows +# use_windows_codec_binaries: True, ### Any relative paths will be interpreted relative to the working directory the script is called from! ### Usage of absolute paths is recommended. -- GitLab From 576f57c5f748870b38be01677cef717b2885f3b2 Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 24 May 2023 17:32:01 +0200 Subject: [PATCH 4/6] change to native binaries as default --- examples/TEMPLATE.yml | 2 +- experiments/selection/BS1534-1a/config/BS1534-1a.yml | 1 + experiments/selection/BS1534-2a/config/BS1534-2a.yml | 1 + experiments/selection/BS1534-2b/config/BS1534-2b.yml | 1 + experiments/selection/BS1534-3a/config/BS1534-3a.yml | 1 + experiments/selection/BS1534-3b/config/BS1534-3b.yml | 1 + experiments/selection/BS1534-4a/config/BS1534-4a.yml | 1 + experiments/selection/BS1534-4b/config/BS1534-4b.yml | 1 + experiments/selection/BS1534-5a/config/BS1534-5a.yml | 1 + experiments/selection/BS1534-5b/config/BS1534-5b.yml | 1 + experiments/selection/BS1534-6a/config/BS1534-6a.yml | 1 + experiments/selection/BS1534-6b/config/BS1534-6b.yml | 1 + experiments/selection/P800-1/config/P800-1.yml | 1 + experiments/selection/P800-2/config/P800-2.yml | 1 + experiments/selection/P800-3/config/P800-3.yml | 1 + experiments/selection/P800-4/config/P800-4.yml | 1 + experiments/selection/P800-5/config/P800-5.yml | 1 + experiments/selection/P800-6/config/P800-6.yml | 1 + experiments/selection/P800-7/config/P800-7.yml | 1 + ivas_processing_scripts/constants.py | 2 +- 20 files changed, 20 insertions(+), 2 deletions(-) diff --git a/examples/TEMPLATE.yml b/examples/TEMPLATE.yml index 9b5889ae..083ac4bb 100755 --- a/examples/TEMPLATE.yml +++ b/examples/TEMPLATE.yml @@ -18,7 +18,7 @@ # master_seed: 5 ### Additional seed to specify number of preruns (used for background noise delay and FER bitstream processing); default = 0 # prerun_seed: 2 -### flag for linux to use windows-built binaries with wine: default = true +### flag for linux to use windows-built binaries with wine: default = false ### this requires the wine binary to be available and will be ignored on windows # use_windows_codec_binaries: True, diff --git a/experiments/selection/BS1534-1a/config/BS1534-1a.yml b/experiments/selection/BS1534-1a/config/BS1534-1a.yml index 22bf59ec..e2c495c1 100644 --- a/experiments/selection/BS1534-1a/config/BS1534-1a.yml +++ b/experiments/selection/BS1534-1a/config/BS1534-1a.yml @@ -9,6 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/BS1534-1a/proc_input" output_path: "experiments/selection/BS1534-1a/proc_output" +use_windows_codec_binaries: True, ################################################ ### Input configuration diff --git a/experiments/selection/BS1534-2a/config/BS1534-2a.yml b/experiments/selection/BS1534-2a/config/BS1534-2a.yml index ad09975a..23411a70 100644 --- a/experiments/selection/BS1534-2a/config/BS1534-2a.yml +++ b/experiments/selection/BS1534-2a/config/BS1534-2a.yml @@ -9,6 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/BS1534-2a/proc_input" output_path: "experiments/selection/BS1534-2a/proc_output" +use_windows_codec_binaries: True, ################################################ ### Input configuration diff --git a/experiments/selection/BS1534-2b/config/BS1534-2b.yml b/experiments/selection/BS1534-2b/config/BS1534-2b.yml index 84ad3bc6..212c3d48 100644 --- a/experiments/selection/BS1534-2b/config/BS1534-2b.yml +++ b/experiments/selection/BS1534-2b/config/BS1534-2b.yml @@ -9,6 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/BS1534-2b/proc_input" output_path: "experiments/selection/BS1534-2b/proc_output" +use_windows_codec_binaries: True, ################################################ ### Input configuration diff --git a/experiments/selection/BS1534-3a/config/BS1534-3a.yml b/experiments/selection/BS1534-3a/config/BS1534-3a.yml index 9285c328..feea203c 100644 --- a/experiments/selection/BS1534-3a/config/BS1534-3a.yml +++ b/experiments/selection/BS1534-3a/config/BS1534-3a.yml @@ -9,6 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/BS1534-3a/proc_input" output_path: "experiments/selection/BS1534-3a/proc_output" +use_windows_codec_binaries: True, ################################################ ### Input configuration diff --git a/experiments/selection/BS1534-3b/config/BS1534-3b.yml b/experiments/selection/BS1534-3b/config/BS1534-3b.yml index 14f877fc..f4252be3 100644 --- a/experiments/selection/BS1534-3b/config/BS1534-3b.yml +++ b/experiments/selection/BS1534-3b/config/BS1534-3b.yml @@ -9,6 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/BS1534-3b/proc_input" output_path: "experiments/selection/BS1534-3b/proc_output" +use_windows_codec_binaries: True, ################################################ ### Input configuration diff --git a/experiments/selection/BS1534-4a/config/BS1534-4a.yml b/experiments/selection/BS1534-4a/config/BS1534-4a.yml index 73b4264f..eebcdcd7 100644 --- a/experiments/selection/BS1534-4a/config/BS1534-4a.yml +++ b/experiments/selection/BS1534-4a/config/BS1534-4a.yml @@ -9,6 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/BS1534-4a/proc_input" output_path: "experiments/selection/BS1534-4a/proc_output" +use_windows_codec_binaries: True, ################################################ ### Input configuration diff --git a/experiments/selection/BS1534-4b/config/BS1534-4b.yml b/experiments/selection/BS1534-4b/config/BS1534-4b.yml index c24a219a..e4ac959d 100644 --- a/experiments/selection/BS1534-4b/config/BS1534-4b.yml +++ b/experiments/selection/BS1534-4b/config/BS1534-4b.yml @@ -9,6 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/BS1534-4b/proc_input" output_path: "experiments/selection/BS1534-4b/proc_output" +use_windows_codec_binaries: True, ################################################ ### Input configuration diff --git a/experiments/selection/BS1534-5a/config/BS1534-5a.yml b/experiments/selection/BS1534-5a/config/BS1534-5a.yml index e9e48f8b..3885d3a5 100644 --- a/experiments/selection/BS1534-5a/config/BS1534-5a.yml +++ b/experiments/selection/BS1534-5a/config/BS1534-5a.yml @@ -9,6 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/BS1534-5a/proc_input" output_path: "experiments/selection/BS1534-5a/proc_output" +use_windows_codec_binaries: True, ################################################ ### Input configuration diff --git a/experiments/selection/BS1534-5b/config/BS1534-5b.yml b/experiments/selection/BS1534-5b/config/BS1534-5b.yml index eeb4c4dd..50feb9ba 100644 --- a/experiments/selection/BS1534-5b/config/BS1534-5b.yml +++ b/experiments/selection/BS1534-5b/config/BS1534-5b.yml @@ -9,6 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/BS1534-5b/proc_input" output_path: "experiments/selection/BS1534-5b/proc_output" +use_windows_codec_binaries: True, ################################################ ### Input configuration diff --git a/experiments/selection/BS1534-6a/config/BS1534-6a.yml b/experiments/selection/BS1534-6a/config/BS1534-6a.yml index 66d9b043..3ebb3bf5 100644 --- a/experiments/selection/BS1534-6a/config/BS1534-6a.yml +++ b/experiments/selection/BS1534-6a/config/BS1534-6a.yml @@ -9,6 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/BS1534-6a/proc_input" output_path: "experiments/selection/BS1534-6a/proc_output" +use_windows_codec_binaries: True, ################################################ ### Input configuration diff --git a/experiments/selection/BS1534-6b/config/BS1534-6b.yml b/experiments/selection/BS1534-6b/config/BS1534-6b.yml index 4998c7b2..51875bff 100644 --- a/experiments/selection/BS1534-6b/config/BS1534-6b.yml +++ b/experiments/selection/BS1534-6b/config/BS1534-6b.yml @@ -9,6 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/BS1534-6b/proc_input" output_path: "experiments/selection/BS1534-6b/proc_output" +use_windows_codec_binaries: True, ################################################ ### Input configuration diff --git a/experiments/selection/P800-1/config/P800-1.yml b/experiments/selection/P800-1/config/P800-1.yml index bd78848f..ede513f4 100644 --- a/experiments/selection/P800-1/config/P800-1.yml +++ b/experiments/selection/P800-1/config/P800-1.yml @@ -9,6 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/P800-1/proc_input" output_path: "experiments/selection/P800-1/proc_output" +use_windows_codec_binaries: True, ################################################ ### Input configuration diff --git a/experiments/selection/P800-2/config/P800-2.yml b/experiments/selection/P800-2/config/P800-2.yml index af6d38cc..f1d85a3b 100644 --- a/experiments/selection/P800-2/config/P800-2.yml +++ b/experiments/selection/P800-2/config/P800-2.yml @@ -9,6 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/P800-2/proc_input" output_path: "experiments/selection/P800-2/proc_output" +use_windows_codec_binaries: True, ################################################ ### Input configuration diff --git a/experiments/selection/P800-3/config/P800-3.yml b/experiments/selection/P800-3/config/P800-3.yml index feb754a4..f739497f 100644 --- a/experiments/selection/P800-3/config/P800-3.yml +++ b/experiments/selection/P800-3/config/P800-3.yml @@ -9,6 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/P800-3/proc_input" output_path: "experiments/selection/P800-3/proc_output" +use_windows_codec_binaries: True, ################################################ ### Input configuration diff --git a/experiments/selection/P800-4/config/P800-4.yml b/experiments/selection/P800-4/config/P800-4.yml index d48a900e..fd86c903 100644 --- a/experiments/selection/P800-4/config/P800-4.yml +++ b/experiments/selection/P800-4/config/P800-4.yml @@ -9,6 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/P800-4/proc_input" output_path: "experiments/selection/P800-4/proc_output" +use_windows_codec_binaries: True, ################################################ ### Input configuration diff --git a/experiments/selection/P800-5/config/P800-5.yml b/experiments/selection/P800-5/config/P800-5.yml index de93f64d..63da2d63 100644 --- a/experiments/selection/P800-5/config/P800-5.yml +++ b/experiments/selection/P800-5/config/P800-5.yml @@ -9,6 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/P800-5/proc_input" output_path: "experiments/selection/P800-5/proc_output" +use_windows_codec_binaries: True, ################################################ ### Input configuration diff --git a/experiments/selection/P800-6/config/P800-6.yml b/experiments/selection/P800-6/config/P800-6.yml index c1dab2ea..9a71b103 100644 --- a/experiments/selection/P800-6/config/P800-6.yml +++ b/experiments/selection/P800-6/config/P800-6.yml @@ -9,6 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/P800-6/proc_input" output_path: "experiments/selection/P800-6/proc_output" +use_windows_codec_binaries: True, ################################################ ### Input configuration diff --git a/experiments/selection/P800-7/config/P800-7.yml b/experiments/selection/P800-7/config/P800-7.yml index 03f1a48f..1fbdb1b2 100644 --- a/experiments/selection/P800-7/config/P800-7.yml +++ b/experiments/selection/P800-7/config/P800-7.yml @@ -9,6 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/P800-7/proc_input" output_path: "experiments/selection/P800-7/proc_output" +use_windows_codec_binaries: True, ################################################ ### Input configuration diff --git a/ivas_processing_scripts/constants.py b/ivas_processing_scripts/constants.py index 5c6b348a..8700b00e 100755 --- a/ivas_processing_scripts/constants.py +++ b/ivas_processing_scripts/constants.py @@ -58,7 +58,7 @@ DEFAULT_CONFIG = { "date": f"{datetime.now().strftime('%Y%m%d_%H.%M.%S')}", "git_sha": f"{get_gitsha()}", "multiprocessing": True, - "use_windows_codec_binaries": True, + "use_windows_codec_binaries": False, "delete_tmp": False, "master_seed": 0, "prerun_seed": 0, -- GitLab From 172561399d921cc5a6ad0a8f3c5438c57e06772f Mon Sep 17 00:00:00 2001 From: Jan Kiene Date: Wed, 24 May 2023 18:28:33 +0200 Subject: [PATCH 5/6] fix configs and get flag from file config first --- experiments/selection/BS1534-1a/config/BS1534-1a.yml | 2 +- experiments/selection/BS1534-1b/config/BS1534-1b.yml | 1 + experiments/selection/BS1534-2a/config/BS1534-2a.yml | 2 +- experiments/selection/BS1534-2b/config/BS1534-2b.yml | 2 +- experiments/selection/BS1534-3a/config/BS1534-3a.yml | 2 +- experiments/selection/BS1534-3b/config/BS1534-3b.yml | 2 +- experiments/selection/BS1534-4a/config/BS1534-4a.yml | 2 +- experiments/selection/BS1534-4b/config/BS1534-4b.yml | 2 +- experiments/selection/BS1534-5a/config/BS1534-5a.yml | 2 +- experiments/selection/BS1534-5b/config/BS1534-5b.yml | 2 +- experiments/selection/BS1534-6a/config/BS1534-6a.yml | 2 +- experiments/selection/BS1534-6b/config/BS1534-6b.yml | 2 +- experiments/selection/P800-1/config/P800-1.yml | 2 +- experiments/selection/P800-2/config/P800-2.yml | 2 +- experiments/selection/P800-3/config/P800-3.yml | 2 +- experiments/selection/P800-4/config/P800-4.yml | 2 +- experiments/selection/P800-5/config/P800-5.yml | 2 +- experiments/selection/P800-6/config/P800-6.yml | 2 +- experiments/selection/P800-7/config/P800-7.yml | 2 +- ivas_processing_scripts/processing/config.py | 3 ++- 20 files changed, 21 insertions(+), 19 deletions(-) diff --git a/experiments/selection/BS1534-1a/config/BS1534-1a.yml b/experiments/selection/BS1534-1a/config/BS1534-1a.yml index e2c495c1..1c84be9f 100644 --- a/experiments/selection/BS1534-1a/config/BS1534-1a.yml +++ b/experiments/selection/BS1534-1a/config/BS1534-1a.yml @@ -9,7 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/BS1534-1a/proc_input" output_path: "experiments/selection/BS1534-1a/proc_output" -use_windows_codec_binaries: True, +use_windows_codec_binaries: True ################################################ ### Input configuration diff --git a/experiments/selection/BS1534-1b/config/BS1534-1b.yml b/experiments/selection/BS1534-1b/config/BS1534-1b.yml index f1806aea..8f2f9869 100644 --- a/experiments/selection/BS1534-1b/config/BS1534-1b.yml +++ b/experiments/selection/BS1534-1b/config/BS1534-1b.yml @@ -9,6 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/BS1534-1b/proc_input" output_path: "experiments/selection/BS1534-1b/proc_output" +use_windows_codec_binaries: True ################################################ ### Input configuration diff --git a/experiments/selection/BS1534-2a/config/BS1534-2a.yml b/experiments/selection/BS1534-2a/config/BS1534-2a.yml index 23411a70..d10ff8ce 100644 --- a/experiments/selection/BS1534-2a/config/BS1534-2a.yml +++ b/experiments/selection/BS1534-2a/config/BS1534-2a.yml @@ -9,7 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/BS1534-2a/proc_input" output_path: "experiments/selection/BS1534-2a/proc_output" -use_windows_codec_binaries: True, +use_windows_codec_binaries: True ################################################ ### Input configuration diff --git a/experiments/selection/BS1534-2b/config/BS1534-2b.yml b/experiments/selection/BS1534-2b/config/BS1534-2b.yml index 212c3d48..7340490f 100644 --- a/experiments/selection/BS1534-2b/config/BS1534-2b.yml +++ b/experiments/selection/BS1534-2b/config/BS1534-2b.yml @@ -9,7 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/BS1534-2b/proc_input" output_path: "experiments/selection/BS1534-2b/proc_output" -use_windows_codec_binaries: True, +use_windows_codec_binaries: True ################################################ ### Input configuration diff --git a/experiments/selection/BS1534-3a/config/BS1534-3a.yml b/experiments/selection/BS1534-3a/config/BS1534-3a.yml index feea203c..dad8771b 100644 --- a/experiments/selection/BS1534-3a/config/BS1534-3a.yml +++ b/experiments/selection/BS1534-3a/config/BS1534-3a.yml @@ -9,7 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/BS1534-3a/proc_input" output_path: "experiments/selection/BS1534-3a/proc_output" -use_windows_codec_binaries: True, +use_windows_codec_binaries: True ################################################ ### Input configuration diff --git a/experiments/selection/BS1534-3b/config/BS1534-3b.yml b/experiments/selection/BS1534-3b/config/BS1534-3b.yml index f4252be3..9bc83f0b 100644 --- a/experiments/selection/BS1534-3b/config/BS1534-3b.yml +++ b/experiments/selection/BS1534-3b/config/BS1534-3b.yml @@ -9,7 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/BS1534-3b/proc_input" output_path: "experiments/selection/BS1534-3b/proc_output" -use_windows_codec_binaries: True, +use_windows_codec_binaries: True ################################################ ### Input configuration diff --git a/experiments/selection/BS1534-4a/config/BS1534-4a.yml b/experiments/selection/BS1534-4a/config/BS1534-4a.yml index eebcdcd7..bbafb275 100644 --- a/experiments/selection/BS1534-4a/config/BS1534-4a.yml +++ b/experiments/selection/BS1534-4a/config/BS1534-4a.yml @@ -9,7 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/BS1534-4a/proc_input" output_path: "experiments/selection/BS1534-4a/proc_output" -use_windows_codec_binaries: True, +use_windows_codec_binaries: True ################################################ ### Input configuration diff --git a/experiments/selection/BS1534-4b/config/BS1534-4b.yml b/experiments/selection/BS1534-4b/config/BS1534-4b.yml index e4ac959d..27e23869 100644 --- a/experiments/selection/BS1534-4b/config/BS1534-4b.yml +++ b/experiments/selection/BS1534-4b/config/BS1534-4b.yml @@ -9,7 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/BS1534-4b/proc_input" output_path: "experiments/selection/BS1534-4b/proc_output" -use_windows_codec_binaries: True, +use_windows_codec_binaries: True ################################################ ### Input configuration diff --git a/experiments/selection/BS1534-5a/config/BS1534-5a.yml b/experiments/selection/BS1534-5a/config/BS1534-5a.yml index 3885d3a5..6e6b313f 100644 --- a/experiments/selection/BS1534-5a/config/BS1534-5a.yml +++ b/experiments/selection/BS1534-5a/config/BS1534-5a.yml @@ -9,7 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/BS1534-5a/proc_input" output_path: "experiments/selection/BS1534-5a/proc_output" -use_windows_codec_binaries: True, +use_windows_codec_binaries: True ################################################ ### Input configuration diff --git a/experiments/selection/BS1534-5b/config/BS1534-5b.yml b/experiments/selection/BS1534-5b/config/BS1534-5b.yml index 50feb9ba..021418ab 100644 --- a/experiments/selection/BS1534-5b/config/BS1534-5b.yml +++ b/experiments/selection/BS1534-5b/config/BS1534-5b.yml @@ -9,7 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/BS1534-5b/proc_input" output_path: "experiments/selection/BS1534-5b/proc_output" -use_windows_codec_binaries: True, +use_windows_codec_binaries: True ################################################ ### Input configuration diff --git a/experiments/selection/BS1534-6a/config/BS1534-6a.yml b/experiments/selection/BS1534-6a/config/BS1534-6a.yml index 3ebb3bf5..eb27ea94 100644 --- a/experiments/selection/BS1534-6a/config/BS1534-6a.yml +++ b/experiments/selection/BS1534-6a/config/BS1534-6a.yml @@ -9,7 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/BS1534-6a/proc_input" output_path: "experiments/selection/BS1534-6a/proc_output" -use_windows_codec_binaries: True, +use_windows_codec_binaries: True ################################################ ### Input configuration diff --git a/experiments/selection/BS1534-6b/config/BS1534-6b.yml b/experiments/selection/BS1534-6b/config/BS1534-6b.yml index 51875bff..1f7b5409 100644 --- a/experiments/selection/BS1534-6b/config/BS1534-6b.yml +++ b/experiments/selection/BS1534-6b/config/BS1534-6b.yml @@ -9,7 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/BS1534-6b/proc_input" output_path: "experiments/selection/BS1534-6b/proc_output" -use_windows_codec_binaries: True, +use_windows_codec_binaries: True ################################################ ### Input configuration diff --git a/experiments/selection/P800-1/config/P800-1.yml b/experiments/selection/P800-1/config/P800-1.yml index ede513f4..e810f021 100644 --- a/experiments/selection/P800-1/config/P800-1.yml +++ b/experiments/selection/P800-1/config/P800-1.yml @@ -9,7 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/P800-1/proc_input" output_path: "experiments/selection/P800-1/proc_output" -use_windows_codec_binaries: True, +use_windows_codec_binaries: True ################################################ ### Input configuration diff --git a/experiments/selection/P800-2/config/P800-2.yml b/experiments/selection/P800-2/config/P800-2.yml index f1d85a3b..22d07037 100644 --- a/experiments/selection/P800-2/config/P800-2.yml +++ b/experiments/selection/P800-2/config/P800-2.yml @@ -9,7 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/P800-2/proc_input" output_path: "experiments/selection/P800-2/proc_output" -use_windows_codec_binaries: True, +use_windows_codec_binaries: True ################################################ ### Input configuration diff --git a/experiments/selection/P800-3/config/P800-3.yml b/experiments/selection/P800-3/config/P800-3.yml index f739497f..696d5a9e 100644 --- a/experiments/selection/P800-3/config/P800-3.yml +++ b/experiments/selection/P800-3/config/P800-3.yml @@ -9,7 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/P800-3/proc_input" output_path: "experiments/selection/P800-3/proc_output" -use_windows_codec_binaries: True, +use_windows_codec_binaries: True ################################################ ### Input configuration diff --git a/experiments/selection/P800-4/config/P800-4.yml b/experiments/selection/P800-4/config/P800-4.yml index fd86c903..9c305937 100644 --- a/experiments/selection/P800-4/config/P800-4.yml +++ b/experiments/selection/P800-4/config/P800-4.yml @@ -9,7 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/P800-4/proc_input" output_path: "experiments/selection/P800-4/proc_output" -use_windows_codec_binaries: True, +use_windows_codec_binaries: True ################################################ ### Input configuration diff --git a/experiments/selection/P800-5/config/P800-5.yml b/experiments/selection/P800-5/config/P800-5.yml index 63da2d63..2bc3d2b2 100644 --- a/experiments/selection/P800-5/config/P800-5.yml +++ b/experiments/selection/P800-5/config/P800-5.yml @@ -9,7 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/P800-5/proc_input" output_path: "experiments/selection/P800-5/proc_output" -use_windows_codec_binaries: True, +use_windows_codec_binaries: True ################################################ ### Input configuration diff --git a/experiments/selection/P800-6/config/P800-6.yml b/experiments/selection/P800-6/config/P800-6.yml index 9a71b103..adabe89c 100644 --- a/experiments/selection/P800-6/config/P800-6.yml +++ b/experiments/selection/P800-6/config/P800-6.yml @@ -9,7 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/P800-6/proc_input" output_path: "experiments/selection/P800-6/proc_output" -use_windows_codec_binaries: True, +use_windows_codec_binaries: True ################################################ ### Input configuration diff --git a/experiments/selection/P800-7/config/P800-7.yml b/experiments/selection/P800-7/config/P800-7.yml index 1fbdb1b2..4427e46b 100644 --- a/experiments/selection/P800-7/config/P800-7.yml +++ b/experiments/selection/P800-7/config/P800-7.yml @@ -9,7 +9,7 @@ prerun_seed: 2 input_path: "experiments/selection/P800-7/proc_input" output_path: "experiments/selection/P800-7/proc_output" -use_windows_codec_binaries: True, +use_windows_codec_binaries: True ################################################ ### Input configuration diff --git a/ivas_processing_scripts/processing/config.py b/ivas_processing_scripts/processing/config.py index f6a26bd0..d2089401 100755 --- a/ivas_processing_scripts/processing/config.py +++ b/ivas_processing_scripts/processing/config.py @@ -96,7 +96,8 @@ class TestConfig: file_cfg = self._parse_yaml(filename) # validate configuration from file - self._validate_file_cfg(file_cfg, cfg["use_windows_codec_binaries"]) + use_windows_codec_binaries = file_cfg.get("use_windows_codec_binaries", cfg["use_windows_codec_binaries"]) + self._validate_file_cfg(file_cfg, use_windows_codec_binaries) # merge dictionaries, overriding from config file merge_dicts(cfg, file_cfg) -- GitLab From 831cf03e399767bdeb089a0cb8ff08cbc017377f Mon Sep 17 00:00:00 2001 From: knj Date: Thu, 25 May 2023 09:49:32 +0200 Subject: [PATCH 6/6] fix formatting --- ivas_processing_scripts/constants.py | 2 +- .../generation/process_ism_items.py | 20 +++++++++++++---- ivas_processing_scripts/processing/config.py | 22 ++++++++++++++----- ivas_processing_scripts/processing/evs.py | 8 +++++-- ivas_processing_scripts/processing/ivas.py | 8 +++++-- 5 files changed, 45 insertions(+), 15 deletions(-) diff --git a/ivas_processing_scripts/constants.py b/ivas_processing_scripts/constants.py index 8700b00e..5cc47609 100755 --- a/ivas_processing_scripts/constants.py +++ b/ivas_processing_scripts/constants.py @@ -33,7 +33,7 @@ from datetime import datetime from pathlib import Path -from ivas_processing_scripts.utils import find_binary, get_binary_paths, get_gitsha +from ivas_processing_scripts.utils import get_binary_paths, get_gitsha LOGGER_SUFFIX = ".log" LOGGER_FORMAT = ( diff --git a/ivas_processing_scripts/generation/process_ism_items.py b/ivas_processing_scripts/generation/process_ism_items.py index cb8a46b8..ff123663 100644 --- a/ivas_processing_scripts/generation/process_ism_items.py +++ b/ivas_processing_scripts/generation/process_ism_items.py @@ -102,10 +102,22 @@ def generate_ism_items( # repeat for all source files for i in range(N_sources): # parse parameters from the scene description - source_file = scene["source"][i] if isinstance(scene["source"], list) else scene["source"] - source_azi = scene["azimuth"][i] if isinstance(scene["azimuth"], list) else scene["azimuth"] - source_ele = scene["elevation"][i] if isinstance(scene["elevation"], list) else scene["elevation"] - + source_file = ( + scene["source"][i] + if isinstance(scene["source"], list) + else scene["source"] + ) + source_azi = ( + scene["azimuth"][i] + if isinstance(scene["azimuth"], list) + else scene["azimuth"] + ) + source_ele = ( + scene["elevation"][i] + if isinstance(scene["elevation"], list) + else scene["elevation"] + ) + logger.info( f"Encoding {source_file} at position(s) {source_azi},{source_ele}" ) diff --git a/ivas_processing_scripts/processing/config.py b/ivas_processing_scripts/processing/config.py index d2089401..09776a79 100755 --- a/ivas_processing_scripts/processing/config.py +++ b/ivas_processing_scripts/processing/config.py @@ -30,11 +30,11 @@ # the United Nations Convention on Contracts on the International Sales of Goods. # +import platform from copy import deepcopy from pathlib import Path import yaml -import platform from ivas_processing_scripts.constants import ( DEFAULT_CONFIG, @@ -45,7 +45,7 @@ from ivas_processing_scripts.constants import ( REQUIRED_KEYS_MNRU, SUPPORTED_CONDITIONS, ) -from ivas_processing_scripts.utils import get_abs_path, find_binary +from ivas_processing_scripts.utils import find_binary, get_abs_path def merge_dicts(base: dict, other: dict) -> None: @@ -96,7 +96,9 @@ class TestConfig: file_cfg = self._parse_yaml(filename) # validate configuration from file - use_windows_codec_binaries = file_cfg.get("use_windows_codec_binaries", cfg["use_windows_codec_binaries"]) + use_windows_codec_binaries = file_cfg.get( + "use_windows_codec_binaries", cfg["use_windows_codec_binaries"] + ) self._validate_file_cfg(file_cfg, use_windows_codec_binaries) # merge dictionaries, overriding from config file @@ -186,7 +188,11 @@ class TestConfig: "Background noise file has to be placed outside the input folder!" ) - codec_bin_extension = ".exe" if platform.system() == "Linux" and use_windows_codec_binaries else "" + codec_bin_extension = ( + ".exe" + if platform.system() == "Linux" and use_windows_codec_binaries + else "" + ) for cond_name, cond_cfg in cfg.get("conditions_to_generate").items(): type = cond_cfg.get("type") if not type: @@ -234,5 +240,9 @@ class TestConfig: def _validate_merged_config(self, cfg: dict): # if not on windows, but "use_windows_codec_binaries" is given, assure that wine is there - if platform.system() == "Linux" and cfg["use_windows_codec_binaries"] and find_binary("wine") is None: - raise FileNotFoundError("Using windows binaries on Linux requires wine") \ No newline at end of file + if ( + platform.system() == "Linux" + and cfg["use_windows_codec_binaries"] + and find_binary("wine") is None + ): + raise FileNotFoundError("Using windows binaries on Linux requires wine") diff --git a/ivas_processing_scripts/processing/evs.py b/ivas_processing_scripts/processing/evs.py index 2871ac85..d99a6d66 100755 --- a/ivas_processing_scripts/processing/evs.py +++ b/ivas_processing_scripts/processing/evs.py @@ -64,12 +64,16 @@ class EVS(Processing): self.name = "evs" self.in_fmt = audio.fromtype(self.in_fmt) self._validate() - self._use_wine = platform.system() == "Linux" and self.use_windows_codec_binaries + self._use_wine = ( + platform.system() == "Linux" and self.use_windows_codec_binaries + ) def _validate( self, ): - need_exe_suffix = platform.system() == "Windows" or self.use_windows_codec_binaries + need_exe_suffix = ( + platform.system() == "Windows" or self.use_windows_codec_binaries + ) if not self.cod_bin or not Path(self.cod_bin).exists(): if need_exe_suffix and ( self.cod_bin and Path(self.cod_bin).with_suffix(".exe").exists() diff --git a/ivas_processing_scripts/processing/ivas.py b/ivas_processing_scripts/processing/ivas.py index b7c70c7a..4f85054e 100755 --- a/ivas_processing_scripts/processing/ivas.py +++ b/ivas_processing_scripts/processing/ivas.py @@ -60,10 +60,14 @@ class IVAS(Processing): self.out_fmt = audio.fromtype(self.out_fmt) if not hasattr(self, "dec_opts"): self.dec_opts = None - self._use_wine = platform.system() == "Linux" and self.use_windows_codec_binaries + self._use_wine = ( + platform.system() == "Linux" and self.use_windows_codec_binaries + ) def _validate(self): - need_exe_suffix = platform.system() == "Windows" or self.use_windows_codec_binaries + need_exe_suffix = ( + platform.system() == "Windows" or self.use_windows_codec_binaries + ) if not self.cod_bin or not Path(self.cod_bin).exists(): if need_exe_suffix and ( self.cod_bin and Path(self.cod_bin).with_suffix(".exe").exists() -- GitLab