From be07ae4e6cc1519ce3c685156e9f235d78d14601 Mon Sep 17 00:00:00 2001 From: veeravt Date: Wed, 19 Apr 2023 12:44:35 +0200 Subject: [PATCH 1/2] Added a function to reorder the items list bsed on concatenation order and made the corresponding changes. --- ivas_processing_scripts/constants.py | 1 + ivas_processing_scripts/processing/processing.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/ivas_processing_scripts/constants.py b/ivas_processing_scripts/constants.py index ff56b9bf..3c4cff91 100755 --- a/ivas_processing_scripts/constants.py +++ b/ivas_processing_scripts/constants.py @@ -59,6 +59,7 @@ DEFAULT_CONFIG = { "multiprocessing": True, "delete_tmp": False, "concatenate_input": False, + "concatenation_order": None, "concat_silence": { "pre": 0, "post": 0, diff --git a/ivas_processing_scripts/processing/processing.py b/ivas_processing_scripts/processing/processing.py index 411f3279..c96251b7 100755 --- a/ivas_processing_scripts/processing/processing.py +++ b/ivas_processing_scripts/processing/processing.py @@ -59,7 +59,23 @@ class Processing(ABC): pass +def reorder_files(items_list: list, concatenation_order: list) -> list: + name_to_full = {Path(full_file).name: full_file for full_file in items_list} + ordered_full_files = [ + name_to_full[name] for name in concatenation_order if name in name_to_full + ] + return ordered_full_files + + def concat_setup(cfg: TestConfig, logger: logging.Logger): + n_items_list = len(cfg.items_list) + if cfg.concatenation_order is not None: + n_concatenation_order = len(cfg.concatenation_order) + if n_concatenation_order != n_items_list: + warn( + f"Warning: Mismatch in specified concatenation order and number of items to process!\nNumber of items specified in concatenation order: {n_concatenation_order}\nNumber of items in the directory: {n_items_list}\nConcatenation will use the following order:\n{cfg.concatenation_order}" + ) + cfg.items_list = reorder_files(cfg.items_list, cfg.concatenation_order) if any([i for i in cfg.items_list if i.suffix == ".txt"]): raise SystemExit("Concatenation for text files is unsupported") -- GitLab From e7f0fb2807df86de79f507f12b286c56467bf7bd Mon Sep 17 00:00:00 2001 From: veeravt Date: Wed, 19 Apr 2023 16:47:48 +0200 Subject: [PATCH 2/2] Took care of ISM by reordering the items list before generating ISM metadata_path. --- examples/TEMPLATE.yml | 4 ++++ ivas_processing_scripts/__init__.py | 4 ++++ ivas_processing_scripts/processing/processing.py | 3 +-- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/examples/TEMPLATE.yml b/examples/TEMPLATE.yml index c63114e3..11f5f8d4 100755 --- a/examples/TEMPLATE.yml +++ b/examples/TEMPLATE.yml @@ -47,6 +47,10 @@ output_path: "./tmp_output" ### Horizontally concatenate input items into one long file; default = false # concatenate_input: true +### Specify the concatenation order in a list of strings. If not specified, the concatenation order would be +### as per the filesystem on the users' device +### Should only be used if concatenate_input = true +# concatenation_order: [] ### Specify preamble duration in ms; default = 0 # preamble: 40 ### Flag wheter to use noise (amplitude +-4) for the preamble or silence; default = false (silence) diff --git a/ivas_processing_scripts/__init__.py b/ivas_processing_scripts/__init__.py index 6d48d5d5..c2cdef05 100755 --- a/ivas_processing_scripts/__init__.py +++ b/ivas_processing_scripts/__init__.py @@ -47,6 +47,7 @@ from ivas_processing_scripts.processing.processing import ( concat_teardown, preprocess, process_item, + reorder_items_list, ) from ivas_processing_scripts.utils import DirManager, apply_func_parallel @@ -92,6 +93,9 @@ def main(args): # set up logging logger = logging_init(args, cfg) + # Re-ordering items based on concatenation order + if cfg.concatenate_input and cfg.concatenation_order is not None: + cfg.items_list = reorder_items_list(cfg.items_list, cfg.concatenation_order) # check for ISM metadata if cfg.input["fmt"].startswith("ISM"): metadata = check_ISM_metadata( diff --git a/ivas_processing_scripts/processing/processing.py b/ivas_processing_scripts/processing/processing.py index c96251b7..7b9a7f86 100755 --- a/ivas_processing_scripts/processing/processing.py +++ b/ivas_processing_scripts/processing/processing.py @@ -59,7 +59,7 @@ class Processing(ABC): pass -def reorder_files(items_list: list, concatenation_order: list) -> list: +def reorder_items_list(items_list: list, concatenation_order: list) -> list: name_to_full = {Path(full_file).name: full_file for full_file in items_list} ordered_full_files = [ name_to_full[name] for name in concatenation_order if name in name_to_full @@ -75,7 +75,6 @@ def concat_setup(cfg: TestConfig, logger: logging.Logger): warn( f"Warning: Mismatch in specified concatenation order and number of items to process!\nNumber of items specified in concatenation order: {n_concatenation_order}\nNumber of items in the directory: {n_items_list}\nConcatenation will use the following order:\n{cfg.concatenation_order}" ) - cfg.items_list = reorder_files(cfg.items_list, cfg.concatenation_order) if any([i for i in cfg.items_list if i.suffix == ".txt"]): raise SystemExit("Concatenation for text files is unsupported") -- GitLab