Commit 4098f8b6 authored by Vinit Veera's avatar Vinit Veera
Browse files

Merge branch 'user-configurable-concatenation-order' into 'main' 

Added a function to reorder the items list based on concatenation order

See merge request !9
parents 6c1a36e3 e7f0fb28

examples/TEMPLATE.yml

+4 −0
Original line number Diff line number Diff line
@@ -51,6 +51,10 @@ output_path: "./tmp_output" 


### Horizontally concatenate input items into one long file; default = false

# concatenate_input: true

### Specify the concatenation order in a list of strings. If not specified, the concatenation order would be

### as per the filesystem on the users' device

### Should only be used if concatenate_input = true

# concatenation_order: []

### Specify preamble duration in ms; default = 0

# preamble: 40

### Flag wheter to use noise (amplitude +-4) for the preamble or silence; default = false (silence)

ivas_processing_scripts/__init__.py

+4 −0
Original line number Diff line number Diff line
@@ -48,6 +48,7 @@ from ivas_processing_scripts.processing.processing import ( 
    concat_teardown,

    preprocess,

    process_item,

    reorder_items_list,

)

from ivas_processing_scripts.utils import DirManager, apply_func_parallel
@@ -93,6 +94,9 @@ def main(args): 
        # set up logging

        logger = logging_init(args, cfg)



        # Re-ordering items based on concatenation order

        if cfg.concatenate_input and cfg.concatenation_order is not None:

            cfg.items_list = reorder_items_list(cfg.items_list, cfg.concatenation_order)

        # check for ISM metadata

        if cfg.input["fmt"].startswith("ISM"):

            metadata = check_ISM_metadata(

ivas_processing_scripts/constants.py

+1 −0
Original line number Diff line number Diff line
@@ -61,6 +61,7 @@ DEFAULT_CONFIG = { 
    "master_seed": 0,

    "prerun_seed": 0,

    "concatenate_input": False,

    "concatenation_order": None,

    "concat_silence": {

        "pre": 0,

        "post": 0,

ivas_processing_scripts/processing/processing.py

+15 −0
Original line number Diff line number Diff line
@@ -58,7 +58,22 @@ class Processing(ABC): 
        pass





def reorder_items_list(items_list: list, concatenation_order: list) -> list:

    name_to_full = {Path(full_file).name: full_file for full_file in items_list}

    ordered_full_files = [

        name_to_full[name] for name in concatenation_order if name in name_to_full

    ]

    return ordered_full_files





def concat_setup(cfg: TestConfig, logger: logging.Logger):

    n_items_list = len(cfg.items_list)

    if cfg.concatenation_order is not None:

        n_concatenation_order = len(cfg.concatenation_order)

        if n_concatenation_order != n_items_list:

            warn(

                f"Warning: Mismatch in specified concatenation order and number of items to process!\nNumber of items specified in concatenation order: {n_concatenation_order}\nNumber of items in the directory: {n_items_list}\nConcatenation will use the following order:\n{cfg.concatenation_order}"

            )

    if any([i for i in cfg.items_list if i.suffix == ".txt"]):

        raise SystemExit("Concatenation for text files is unsupported")