Commit 0a76ac39 authored by Archit Tamarapu's avatar Archit Tamarapu
Browse files

parallelise conditions instead of items

parent a45a6b0a

ivas_processing_scripts/__init__.py

+10 −21
Original line number Diff line number Diff line
@@ -44,6 +44,7 @@ from ivas_processing_scripts.processing.processing import ( 
    preprocess,

    preprocess_2,

    preprocess_background_noise,

    process_condition,

    process_item,

    reorder_items_list,

)
@@ -147,23 +148,11 @@ def main(args): 
            # preprocess 2

            preprocess_2(cfg, logger)



        # run conditions

        for condition, out_dir, tmp_dir in zip(

            cfg.proc_chains, cfg.out_dirs, cfg.tmp_dirs

        ):

            chain = condition["processes"]



            logger.info(f"  Generating condition: {condition['name']}")



        # parallel processing of conditions

        apply_func_parallel(

                process_item,

            process_condition,

            zip(

                    cfg.items_list,

                    repeat(tmp_dir),

                    repeat(out_dir),

                    repeat(chain),

                    repeat(logger),

                    cfg.metadata_path,

                repeat(cfg), cfg.proc_chains, cfg.tmp_dirs, cfg.out_dirs, repeat(logger)

            ),

            None,

            "mp" if cfg.multiprocessing else None,

ivas_processing_scripts/processing/processing.py

+36 −1
Original line number Diff line number Diff line
@@ -402,6 +402,42 @@ def process_item( 
                copyfile(ppm, out_meta[idx])





def process_condition(

    cfg: TestConfig,

    condition: dict,

    tmp_dir: Union[str, Path],

    out_dir: Union[str, Path],

    logger: logging.Logger,

):

    chain = condition["processes"]



    logger.info(f"  Generating condition: {condition['name']}")



    for item, metadata in zip(cfg.items_list, cfg.metadata_path):

        process_item(

            item,

            tmp_dir,

            out_dir,

            chain,

            logger,

            metadata,

        )

    # apply_func_parallel(

    #     process_item,

    #     zip(

    #         cfg.items_list,

    #         repeat(tmp_dir),

    #         repeat(out_dir),

    #         repeat(chain),

    #         repeat(logger),

    #         cfg.metadata_path,

    #     ),

    #     None,

    #     "mp" if cfg.multiprocessing else None,

    # )

    #





def remove_pre_and_postamble(

    x, out_fmt, fs, repeat_signal, preamble_len_ms, postamble_len_ms, meta, logger

):
@@ -464,4 +500,3 @@ def preprocess_background_noise(cfg): 
    ] = output_audio



    return