diff --git a/ivas_processing_scripts/__init__.py b/ivas_processing_scripts/__init__.py index 15424a3a36d74396fc4bbbcab3025475ace59838..abeb9f4f88971df803f1474d54c15236dcf67789 100755 --- a/ivas_processing_scripts/__init__.py +++ b/ivas_processing_scripts/__init__.py @@ -44,7 +44,7 @@ from ivas_processing_scripts.processing.processing import ( preprocess, preprocess_2, preprocess_background_noise, - process_item, + process_condition, reorder_items_list, ) from ivas_processing_scripts.utils import DirManager, apply_func_parallel @@ -147,27 +147,21 @@ def main(args): # preprocess 2 preprocess_2(cfg, logger) - # run conditions - for condition, out_dir, tmp_dir in zip( - cfg.proc_chains, cfg.out_dirs, cfg.tmp_dirs - ): - chain = condition["processes"] - - logger.info(f" Generating condition: {condition['name']}") - - apply_func_parallel( - process_item, - zip( - cfg.items_list, - repeat(tmp_dir), - repeat(out_dir), - repeat(chain), - repeat(logger), - cfg.metadata_path, - ), - None, - "mp" if cfg.multiprocessing else None, - ) + # attempt to parallelise either items or conditions based on test setup + parallelise_items = len(cfg.items_list) > len(cfg.proc_chains) + apply_func_parallel( + process_condition, + zip( + repeat(cfg), + cfg.proc_chains, + cfg.tmp_dirs, + cfg.out_dirs, + repeat(logger), + repeat(parallelise_items), + ), + None, + "mp" if (not parallelise_items and cfg.multiprocessing) else None, + ) # copy configuration to output directory cfg.to_file(cfg.output_path.joinpath(f"{cfg.name}.yml")) diff --git a/ivas_processing_scripts/generation/__init__.py b/ivas_processing_scripts/generation/__init__.py index 8ee7c026d64269a0de0d7988b9df7ea5424a66e4..53892744063538d89de3137b14c0fe3ddd7f1b06 100755 --- a/ivas_processing_scripts/generation/__init__.py +++ b/ivas_processing_scripts/generation/__init__.py @@ -31,7 +31,6 @@ # import logging -import os import yaml diff --git a/ivas_processing_scripts/processing/processing.py b/ivas_processing_scripts/processing/processing.py index 29797295cc205b6fb5b2ad726211fba53a3d693e..23be60732fb89a39708bc74d79352d0c8f50bdb2 100755 --- a/ivas_processing_scripts/processing/processing.py +++ b/ivas_processing_scripts/processing/processing.py @@ -402,6 +402,44 @@ def process_item( copyfile(ppm, out_meta[idx]) +def process_condition( + cfg: TestConfig, + condition: dict, + tmp_dir: Union[str, Path], + out_dir: Union[str, Path], + logger: logging.Logger, + parallelise_items: bool = False, +): + chain = condition["processes"] + + logger.info(f" Generating condition: {condition['name']}") + + if parallelise_items: + apply_func_parallel( + process_item, + zip( + cfg.items_list, + repeat(tmp_dir), + repeat(out_dir), + repeat(chain), + repeat(logger), + cfg.metadata_path, + ), + None, + "mp" if cfg.multiprocessing else None, + ) + else: + for item, metadata in zip(cfg.items_list, cfg.metadata_path): + process_item( + item, + tmp_dir, + out_dir, + chain, + logger, + metadata, + ) + + def remove_pre_and_postamble( x, out_fmt, fs, repeat_signal, preamble_len_ms, postamble_len_ms, meta, logger ): @@ -464,4 +502,3 @@ def preprocess_background_noise(cfg): ] = output_audio return -